use std::path::{Path, PathBuf};
use anyhow::Context;
use async_trait::async_trait;
use reqwest::header::{AUTHORIZATION, CONTENT_LENGTH, CONTENT_TYPE};
use crate::assets::{
firebase_download_url, urlencoding_encode_path, DEFAULT_FIREBASE_BUCKET,
};
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum AssetBackendKind {
Local,
Gcs,
S3,
}
impl AssetBackendKind {
pub fn parse(raw: &str) -> anyhow::Result<Self> {
match raw.trim().to_ascii_lowercase().as_str() {
"local" | "file" | "disk" => Ok(Self::Local),
"gcs" | "firebase" | "gs" => Ok(Self::Gcs),
"s3" | "minio" | "r2" => Ok(Self::S3),
other => anyhow::bail!(
"unknown FLATLAND_ASSETS_BACKEND={other:?} (expected local|gcs|s3)"
),
}
}
}
#[derive(Debug, Clone)]
pub struct AssetBackendConfig {
pub kind: AssetBackendKind,
pub bucket: String,
pub client_prefix: String,
pub sim_prefix: String,
pub local_root: PathBuf,
pub s3_region: String,
pub s3_endpoint: Option<String>,
pub index_url_override: Option<String>,
}
impl Default for AssetBackendConfig {
fn default() -> Self {
Self::from_env()
}
}
impl AssetBackendConfig {
pub fn from_env() -> Self {
let kind = std::env::var("FLATLAND_ASSETS_BACKEND")
.ok()
.and_then(|s| AssetBackendKind::parse(&s).ok())
.unwrap_or(AssetBackendKind::Gcs);
let bucket = std::env::var("FLATLAND_ASSETS_BUCKET")
.unwrap_or_else(|_| DEFAULT_FIREBASE_BUCKET.to_string());
let client_prefix = std::env::var("FLATLAND_ASSETS_PREFIX")
.unwrap_or_else(|_| "flatland3/client-assets".to_string());
let sim_prefix = std::env::var("FLATLAND_SIM_ASSETS_PREFIX")
.unwrap_or_else(|_| "flatland3/sim-content".to_string());
let local_root = std::env::var("FLATLAND_ASSETS_LOCAL_ROOT")
.map(PathBuf::from)
.unwrap_or_else(|_| {
dirs::home_dir()
.unwrap_or_else(|| PathBuf::from("."))
.join(".flatland3")
.join("published")
});
let s3_region =
std::env::var("FLATLAND_ASSETS_S3_REGION").unwrap_or_else(|_| "us-east-1".to_string());
let s3_endpoint = std::env::var("FLATLAND_ASSETS_S3_ENDPOINT")
.ok()
.filter(|s| !s.is_empty());
let index_url_override = std::env::var("FLATLAND_ASSETS_INDEX_URL")
.ok()
.filter(|s| !s.is_empty());
Self {
kind,
bucket,
client_prefix,
sim_prefix,
local_root,
s3_region,
s3_endpoint,
index_url_override,
}
}
pub fn client_index_object(&self) -> String {
format!("{}/latest.json", self.client_prefix.trim_end_matches('/'))
}
pub fn sim_index_object(&self) -> String {
format!("{}/latest.json", self.sim_prefix.trim_end_matches('/'))
}
pub fn client_index_url(&self) -> String {
if let Some(url) = &self.index_url_override {
return url.clone();
}
self.object_url(&self.client_index_object())
}
pub fn sim_index_url(&self) -> String {
self.object_url(&self.sim_index_object())
}
pub fn client_object_key(&self, publish_rev: u64, relative: &str) -> String {
format!(
"{}/rev-{}/{}",
self.client_prefix.trim_end_matches('/'),
publish_rev,
relative
)
}
pub fn sim_object_key(&self, publish_rev: u64, relative: &str) -> String {
format!(
"{}/rev-{}/{}",
self.sim_prefix.trim_end_matches('/'),
publish_rev,
relative
)
}
pub fn object_url(&self, object_key: &str) -> String {
match self.kind {
AssetBackendKind::Local => {
let path = self.local_root.join(object_key);
format!("file://{}", path.display())
}
AssetBackendKind::Gcs => firebase_download_url(&self.bucket, object_key),
AssetBackendKind::S3 => self.s3_object_url(object_key),
}
}
fn s3_object_url(&self, object_key: &str) -> String {
let key = object_key.trim_start_matches('/');
if let Some(endpoint) = &self.s3_endpoint {
let base = endpoint.trim_end_matches('/');
format!("{base}/{}/{key}", self.bucket)
} else {
format!(
"https://{}.s3.{}.amazonaws.com/{key}",
self.bucket, self.s3_region
)
}
}
pub fn local_object_path(&self, object_key: &str) -> PathBuf {
self.local_root.join(object_key)
}
}
#[async_trait]
pub trait AssetStore: Send + Sync {
async fn put(
&self,
object_key: &str,
content_type: &str,
bytes: &[u8],
) -> anyhow::Result<()>;
async fn get(&self, object_key: &str) -> anyhow::Result<Vec<u8>>;
}
pub fn store_from_config(cfg: &AssetBackendConfig) -> anyhow::Result<Box<dyn AssetStore>> {
match cfg.kind {
AssetBackendKind::Local => Ok(Box::new(LocalAssetStore {
root: cfg.local_root.clone(),
})),
AssetBackendKind::Gcs => Ok(Box::new(GcsAssetStore {
bucket: cfg.bucket.clone(),
client: reqwest::Client::new(),
})),
AssetBackendKind::S3 => Ok(Box::new(S3AssetStore {
bucket: cfg.bucket.clone(),
region: cfg.s3_region.clone(),
endpoint: cfg.s3_endpoint.clone(),
client: reqwest::Client::new(),
})),
}
}
pub struct LocalAssetStore {
pub root: PathBuf,
}
#[async_trait]
impl AssetStore for LocalAssetStore {
async fn put(
&self,
object_key: &str,
_content_type: &str,
bytes: &[u8],
) -> anyhow::Result<()> {
let path = self.root.join(object_key);
if let Some(parent) = path.parent() {
std::fs::create_dir_all(parent)
.with_context(|| format!("mkdir {}", parent.display()))?;
}
std::fs::write(&path, bytes).with_context(|| format!("write {}", path.display()))?;
Ok(())
}
async fn get(&self, object_key: &str) -> anyhow::Result<Vec<u8>> {
let path = self.root.join(object_key);
std::fs::read(&path).with_context(|| format!("read {}", path.display()))
}
}
pub struct GcsAssetStore {
pub bucket: String,
pub client: reqwest::Client,
}
#[async_trait]
impl AssetStore for GcsAssetStore {
async fn put(
&self,
object_key: &str,
content_type: &str,
bytes: &[u8],
) -> anyhow::Result<()> {
let token = gcs_upload_bearer_token().await?;
let url = format!(
"https://storage.googleapis.com/upload/storage/v1/b/{}/o?uploadType=media&name={}",
self.bucket,
urlencoding_encode_path(object_key)
);
let response = self
.client
.post(&url)
.header(AUTHORIZATION, format!("Bearer {token}"))
.header(CONTENT_TYPE, content_type)
.header(CONTENT_LENGTH, bytes.len())
.body(bytes.to_vec())
.send()
.await?;
if !response.status().is_success() {
let status = response.status();
let body = response.text().await.unwrap_or_default();
anyhow::bail!("GCS upload {object_key} failed: {status} {body}");
}
Ok(())
}
async fn get(&self, object_key: &str) -> anyhow::Result<Vec<u8>> {
let url = firebase_download_url(&self.bucket, object_key);
let response = self.client.get(&url).send().await?.error_for_status()?;
Ok(response.bytes().await?.to_vec())
}
}
pub struct S3AssetStore {
pub bucket: String,
pub region: String,
pub endpoint: Option<String>,
pub client: reqwest::Client,
}
#[async_trait]
impl AssetStore for S3AssetStore {
async fn put(
&self,
object_key: &str,
content_type: &str,
bytes: &[u8],
) -> anyhow::Result<()> {
if which_aws_cli() {
return s3_cli_put(
&self.bucket,
object_key,
content_type,
bytes,
self.endpoint.as_deref(),
&self.region,
)
.await;
}
anyhow::bail!(
"S3 upload requires the `aws` CLI (aws s3 cp) or set FLATLAND_ASSETS_BACKEND=gcs|local"
)
}
async fn get(&self, object_key: &str) -> anyhow::Result<Vec<u8>> {
let url = if let Some(endpoint) = &self.endpoint {
format!(
"{}/{}/{}",
endpoint.trim_end_matches('/'),
self.bucket,
object_key
)
} else {
format!(
"https://{}.s3.{}.amazonaws.com/{}",
self.bucket, self.region, object_key
)
};
let response = self.client.get(&url).send().await?.error_for_status()?;
Ok(response.bytes().await?.to_vec())
}
}
fn which_aws_cli() -> bool {
std::process::Command::new("aws")
.arg("--version")
.output()
.map(|o| o.status.success())
.unwrap_or(false)
}
async fn s3_cli_put(
bucket: &str,
object_key: &str,
content_type: &str,
bytes: &[u8],
endpoint: Option<&str>,
region: &str,
) -> anyhow::Result<()> {
let tmp = tempfile_path(object_key)?;
if let Some(parent) = tmp.parent() {
std::fs::create_dir_all(parent)?;
}
std::fs::write(&tmp, bytes)?;
let uri = format!("s3://{bucket}/{object_key}");
let mut cmd = tokio::process::Command::new("aws");
cmd.args(["s3", "cp", tmp.to_str().unwrap(), &uri]);
cmd.args(["--content-type", content_type]);
cmd.args(["--region", region]);
if let Some(ep) = endpoint {
cmd.args(["--endpoint-url", ep]);
}
let output = cmd.output().await?;
let _ = std::fs::remove_file(&tmp);
if !output.status.success() {
anyhow::bail!(
"aws s3 cp failed: {}",
String::from_utf8_lossy(&output.stderr)
);
}
Ok(())
}
fn tempfile_path(object_key: &str) -> anyhow::Result<PathBuf> {
let name = object_key.replace('/', "_");
let dir = std::env::temp_dir().join("flatland-asset-upload");
std::fs::create_dir_all(&dir)?;
Ok(dir.join(name))
}
async fn gcs_upload_bearer_token() -> anyhow::Result<String> {
if let Ok(token) = std::env::var("FLATLAND_ASSETS_UPLOAD_TOKEN") {
if !token.is_empty() {
return Ok(token);
}
}
let output = tokio::process::Command::new("gcloud")
.args(["auth", "application-default", "print-access-token"])
.output()
.await
.map_err(|err| {
anyhow::anyhow!(
"gcloud ADC token failed ({err}); set FLATLAND_ASSETS_UPLOAD_TOKEN or run gcloud auth application-default login"
)
})?;
if !output.status.success() {
anyhow::bail!(
"gcloud auth application-default print-access-token failed: {}",
String::from_utf8_lossy(&output.stderr)
);
}
Ok(String::from_utf8(output.stdout)?.trim().to_string())
}
pub fn mime_for_path(path: &Path) -> &'static str {
match path
.extension()
.and_then(|e| e.to_str())
.unwrap_or("")
.to_ascii_lowercase()
.as_str()
{
"png" => "image/png",
"jpg" | "jpeg" => "image/jpeg",
"webp" => "image/webp",
"json" => "application/json",
"yaml" | "yml" => "application/x-yaml",
_ => "application/octet-stream",
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn parse_backend_kinds() {
assert_eq!(
AssetBackendKind::parse("firebase").unwrap(),
AssetBackendKind::Gcs
);
assert_eq!(
AssetBackendKind::parse("local").unwrap(),
AssetBackendKind::Local
);
assert_eq!(
AssetBackendKind::parse("minio").unwrap(),
AssetBackendKind::S3
);
}
#[test]
fn gcs_object_urls() {
let cfg = AssetBackendConfig {
kind: AssetBackendKind::Gcs,
bucket: "flatland-8911e.appspot.com".into(),
client_prefix: "flatland3/client-assets".into(),
sim_prefix: "flatland3/sim-content".into(),
local_root: PathBuf::from("/tmp"),
s3_region: "us-east-1".into(),
s3_endpoint: None,
index_url_override: None,
};
let url = cfg.client_index_url();
assert!(url.contains("flatland-8911e.appspot.com"));
assert!(url.contains("latest.json"));
assert_eq!(
cfg.client_index_object(),
"flatland3/client-assets/latest.json"
);
}
}