use std::sync::Arc;
use anyhow::{Result, bail};
use dashmap::DashMap;
use dashmap::mapref::entry::Entry;
use crate::buc::store::metered::MeteredObjectStore;
use crate::buc::store::prefixed::PrefixedStore;
use crate::buc::store::{ObjectKey, ObjectStore};
use crate::buc::{BucketStoreProvider, Config, Error};
use crate::catalog::providers::BucketProvider;
use crate::catalog::{DatabaseId, NamespaceId};
use crate::kvs::Transaction;
use crate::observe::ExecutionObserver;
struct Inner {
connections: DashMap<BucketConnectionKey, Arc<dyn ObjectStore>>,
provider: Box<dyn BucketStoreProvider>,
config: Config,
observer: Arc<dyn ExecutionObserver>,
}
fn backend_label(url: &str) -> &'static str {
let scheme = url
.split_once("://")
.map(|(s, _)| s)
.or_else(|| url.split_once(':').map(|(s, _)| s))
.unwrap_or(url);
match scheme {
"s3" | "s3+http" | "s3+https" => "s3",
"gs" | "gcs" => "gcs",
"az" | "azure" => "azure",
"file" => "file",
"memory" => "memory",
_ => "other",
}
}
#[derive(Clone)]
pub(crate) struct BucketsManager(Arc<Inner>);
impl BucketsManager {
pub(crate) fn new(
provider: Box<dyn BucketStoreProvider>,
config: Config,
observer: Arc<dyn ExecutionObserver>,
) -> Self {
BucketsManager(Arc::new(Inner {
connections: DashMap::new(),
provider,
config,
observer,
}))
}
fn meter(&self, store: Arc<dyn ObjectStore>, url: &str) -> Arc<dyn ObjectStore> {
Arc::new(MeteredObjectStore::new(store, Arc::clone(&self.0.observer), backend_label(url)))
}
pub(crate) fn clear(&self) {
self.0.connections.clear();
}
async fn connect(
&self,
url: &str,
global: bool,
readonly: bool,
) -> Result<Arc<dyn ObjectStore>> {
if !global && self.0.config.only_global {
bail!(Error::GlobalBucketEnforced);
}
let store = self.0.provider.connect(url, global, readonly, self.0.config.clone()).await?;
Ok(self.meter(store, url))
}
async fn connect_global(
&self,
ns: NamespaceId,
db: DatabaseId,
bu: &str,
) -> Result<Arc<dyn ObjectStore>> {
let Some(ref url) = self.0.config.global_bucket else {
bail!(Error::NoGlobalBucket);
};
let global = self.0.provider.connect(url, true, false, self.0.config.clone()).await?;
let key = ObjectKey::new(format!("/{ns}/{db}/{bu}"));
let prefixed: Arc<dyn ObjectStore> = Arc::new(PrefixedStore::new(global, key));
Ok(self.meter(prefixed, url))
}
pub(crate) async fn get_bucket_store(
&self,
tx: &Transaction,
ns: NamespaceId,
db: DatabaseId,
bu: &str,
) -> Result<Arc<dyn ObjectStore>> {
let key = BucketConnectionKey::new(ns, db, bu);
if let Some(existing) = self.0.connections.get(&key) {
return Ok(Arc::clone(existing.value()));
}
let bd = tx.expect_db_bucket(ns, db, bu).await?;
let store = if let Some(ref backend) = bd.backend {
self.connect(backend, false, bd.readonly).await?
} else {
self.connect_global(ns, db, bu).await?
};
match self.0.connections.entry(key) {
Entry::Occupied(e) => Ok(Arc::clone(e.get())),
Entry::Vacant(e) => {
e.insert(Arc::clone(&store));
Ok(store)
}
}
}
pub(crate) async fn new_backend(
&self,
ns: NamespaceId,
db: DatabaseId,
bu: &str,
read_only: bool,
backend: Option<&str>,
) -> Result<()> {
let store = if let Some(backend) = backend {
self.connect(backend, false, read_only).await?
} else {
self.connect_global(ns, db, bu).await?
};
let key = BucketConnectionKey::new(ns, db, bu);
self.0.connections.insert(key, store);
Ok(())
}
}
#[derive(Hash, PartialEq, Eq)]
pub(super) struct BucketConnectionKey {
ns: NamespaceId,
db: DatabaseId,
bu: String,
}
impl BucketConnectionKey {
pub fn new(ns: NamespaceId, db: DatabaseId, bu: &str) -> Self {
Self {
ns,
db,
bu: bu.into(),
}
}
}