use std::sync::Arc;
use object_store::azure::MicrosoftAzureBuilder;
use object_store::path::Path;
use object_store::prefix::PrefixStore;
use object_store::{ObjectStore, Result};
use unitycatalog_client::{TemporaryCredentialClient, UnityCatalogClient};
#[cfg(not(target_arch = "wasm32"))]
use object_store::aws::AmazonS3Builder;
#[cfg(not(target_arch = "wasm32"))]
use object_store::client::SpawnedReqwestConnector;
#[cfg(not(target_arch = "wasm32"))]
use object_store::gcp::GoogleCloudStorageBuilder;
#[cfg(not(target_arch = "wasm32"))]
use object_store::local::LocalFileSystem;
#[cfg(not(target_arch = "wasm32"))]
use tokio::runtime::Handle;
#[cfg(not(target_arch = "wasm32"))]
use olai_http::CloudClient as Transport;
#[cfg(not(target_arch = "wasm32"))]
use olai_http::service::{HttpService, ReqwestService};
#[cfg(target_arch = "wasm32")]
use olai_http_wasm::WasmClient as Transport;
use unitycatalog_common::tables::v1::GetTableRequest;
use unitycatalog_common::temporary_credentials::v1::TemporaryCredential;
use unitycatalog_common::volumes::v1::GetVolumeRequest;
use url::Url;
use crate::credential::{SecurableRef, as_aws, as_azure, as_gcp, new_azure};
#[cfg(not(target_arch = "wasm32"))]
use crate::credential::{aws_access_point, new_aws, new_gcp};
pub use crate::error::Error;
pub use unitycatalog_common::UCReference;
pub use unitycatalog_client::{
PathOperation, TableOperation, TableReference, VolumeOperation, VolumeReference,
};
mod credential;
mod error;
mod proxy;
#[cfg(not(target_arch = "wasm32"))]
#[derive(Debug)]
struct ForwardedHeaderService {
inner: Arc<dyn HttpService>,
name: reqwest::header::HeaderName,
value: reqwest::header::HeaderValue,
}
#[cfg(not(target_arch = "wasm32"))]
impl HttpService for ForwardedHeaderService {
fn call(
&self,
mut request: reqwest::Request,
) -> std::pin::Pin<
Box<dyn std::future::Future<Output = olai_http::Result<reqwest::Response>> + Send + '_>,
> {
request
.headers_mut()
.insert(self.name.clone(), self.value.clone());
self.inner.call(request)
}
}
#[derive(Debug, Clone, Default)]
pub struct UnityObjectStoreFactoryBuilder {
uri: Option<String>,
token: Option<String>,
allow_unauthenticated: bool,
aws_region: Option<String>,
#[cfg(not(target_arch = "wasm32"))]
io_handle: Option<Handle>,
}
impl UnityObjectStoreFactoryBuilder {
pub fn new() -> Self {
Self::default()
}
pub fn with_uri(mut self, uri: impl Into<String>) -> Self {
self.uri = Some(uri.into());
self
}
pub fn with_token(mut self, token: impl Into<Option<String>>) -> Self {
self.token = token.into();
self
}
pub fn with_allow_unauthenticated(mut self, allow_unauthenticated: bool) -> Self {
self.allow_unauthenticated = allow_unauthenticated;
self
}
pub fn with_aws_region(mut self, aws_region: impl Into<Option<String>>) -> Self {
self.aws_region = aws_region.into();
self
}
#[cfg(not(target_arch = "wasm32"))]
pub fn with_io_runtime(mut self, handle: impl Into<Option<Handle>>) -> Self {
self.io_handle = handle.into();
self
}
pub async fn build(self) -> Result<UnityObjectStoreFactory> {
let url = if let Some(uri) = self.uri.as_ref() {
url::Url::parse(uri).map_err(Error::from)?
} else {
return Err(Error::invalid_config("missing `uri` for Unity Catalog endpoint").into());
};
let transport = self.build_transport()?;
#[cfg(target_arch = "wasm32")]
let storage_access = proxy::discover_storage_access(&url, self.token.as_deref()).await;
let creds = TemporaryCredentialClient::new_with_url(transport.clone(), url.clone());
let uc = UnityCatalogClient::new(transport, url.clone());
Ok(UnityObjectStoreFactory {
creds,
uc,
aws_region: self.aws_region,
#[cfg(not(target_arch = "wasm32"))]
io_handle: self.io_handle,
#[cfg(not(target_arch = "wasm32"))]
base_url: url,
#[cfg(not(target_arch = "wasm32"))]
allow_unauthenticated: self.allow_unauthenticated,
#[cfg(target_arch = "wasm32")]
storage_access,
#[cfg(target_arch = "wasm32")]
token: self.token,
})
}
#[cfg(not(target_arch = "wasm32"))]
fn build_transport(&self) -> Result<Transport> {
let cloud_client = if let Some(token) = &self.token {
Transport::new_with_token(token)
} else if self.allow_unauthenticated {
Transport::new_unauthenticated()
} else {
return Err(Error::invalid_config(
"no token and `allow_unauthenticated` not set: cannot build credential client",
)
.into());
};
Ok(match &self.io_handle {
Some(handle) => cloud_client.with_runtime(handle.clone()),
None => cloud_client,
})
}
#[cfg(target_arch = "wasm32")]
fn build_transport(&self) -> Result<Transport> {
use olai_http_wasm::CredentialsMode;
use reqwest::header::{AUTHORIZATION, HeaderMap, HeaderValue};
let client = Transport::new();
Ok(match &self.token {
Some(token) => {
let token = token.clone();
client
.with_credentials(CredentialsMode::Omit)
.with_auth(move || {
let mut headers = HeaderMap::new();
if let Ok(value) = HeaderValue::from_str(&format!("Bearer {token}")) {
headers.insert(AUTHORIZATION, value);
}
headers
})
}
None => client,
})
}
}
#[derive(Clone)]
pub struct UCStore {
root: Arc<dyn ObjectStore>,
url: Url,
path: Path,
}
impl UCStore {
pub fn as_dyn(&self) -> Arc<dyn ObjectStore> {
if self.path.as_ref().is_empty() {
self.root.clone()
} else {
Arc::new(PrefixStore::new(self.root.clone(), self.path.clone()))
}
}
pub fn root(&self) -> Arc<dyn ObjectStore> {
self.root.clone()
}
pub fn url(&self) -> &Url {
&self.url
}
pub fn prefix(&self) -> &Path {
&self.path
}
}
#[derive(Clone)]
pub struct UnityObjectStoreFactory {
creds: TemporaryCredentialClient,
uc: UnityCatalogClient,
#[cfg_attr(target_arch = "wasm32", allow(dead_code))]
aws_region: Option<String>,
#[cfg(not(target_arch = "wasm32"))]
io_handle: Option<Handle>,
#[cfg(not(target_arch = "wasm32"))]
base_url: Url,
#[cfg(not(target_arch = "wasm32"))]
#[allow(dead_code)]
allow_unauthenticated: bool,
#[cfg(target_arch = "wasm32")]
storage_access: proxy::StorageAccess,
#[cfg(target_arch = "wasm32")]
token: Option<String>,
}
impl UnityObjectStoreFactory {
pub fn builder() -> UnityObjectStoreFactoryBuilder {
UnityObjectStoreFactoryBuilder::default()
}
#[cfg(not(target_arch = "wasm32"))]
pub fn with_forwarded_user(&self, header: &str, user: Option<&str>) -> Result<Self> {
use reqwest::header::{HeaderName, HeaderValue};
let Some(name) = user else {
return Ok(self.clone());
};
let header_name = HeaderName::from_bytes(header.as_bytes()).map_err(|e| {
Error::invalid_config(format!(
"invalid forwarded-user header name `{header}`: {e}"
))
})?;
let header_value = HeaderValue::from_str(name).map_err(|e| {
Error::invalid_config(format!("invalid forwarded-user header value: {e}"))
})?;
let reqwest_client = reqwest::Client::new();
let base_service: Arc<dyn HttpService> =
Arc::new(ReqwestService::new(reqwest_client.clone()));
let transport =
Transport::new_unauthenticated().with_http_service(Arc::new(ForwardedHeaderService {
inner: base_service,
name: header_name,
value: header_value,
}));
let transport = match &self.io_handle {
Some(handle) => transport.with_runtime(handle.clone()),
None => transport,
};
let creds =
TemporaryCredentialClient::new_with_url(transport.clone(), self.base_url.clone());
let uc = UnityCatalogClient::new(transport, self.base_url.clone());
Ok(UnityObjectStoreFactory {
creds,
uc,
aws_region: self.aws_region.clone(),
io_handle: self.io_handle.clone(),
base_url: self.base_url.clone(),
allow_unauthenticated: self.allow_unauthenticated,
})
}
pub fn unity_client(&self) -> &UnityCatalogClient {
&self.uc
}
pub fn credentials_client(&self) -> &TemporaryCredentialClient {
&self.creds
}
pub async fn for_url(&self, url: &str, op: Operation) -> Result<UCStore> {
let reference = UCReference::parse(url)
.map_err(crate::error::Error::from)
.map_err(object_store::Error::from)?;
match reference {
UCReference::Volume {
catalog,
schema,
volume,
path,
} => {
let name = format!("{catalog}.{schema}.{volume}");
let store = self.for_volume(name, op.into_volume()).await?;
if path.is_empty() {
Ok(store)
} else {
Ok(extend_prefix(store, &path))
}
}
UCReference::Table {
catalog,
schema,
table,
} => {
let name = format!("{catalog}.{schema}.{table}");
self.for_table(name, op.into_table()).await
}
UCReference::Path(url) => self.for_path(&url, op.into_path()).await,
}
}
pub async fn for_table(
&self,
table: impl Into<TableReference>,
operation: TableOperation,
) -> Result<UCStore> {
let table = table.into();
if let TableReference::Name(name) = &table
&& let Some(location) = self.table_storage_location(name).await?
&& let Ok(url) = Url::parse(&location)
{
if url.scheme() == "file" {
let path_op = match operation {
TableOperation::Read => PathOperation::Read,
TableOperation::ReadWrite => PathOperation::ReadWrite,
};
return local_store(&url, path_op);
}
#[cfg(target_arch = "wasm32")]
if matches!(self.storage_access, proxy::StorageAccess::Proxy { .. }) {
let seg = format!("table:{name}");
return self.proxy_store(&seg, &url);
}
}
let (credential, table_id) = self
.creds
.temporary_table_credential(table, operation)
.await
.map_err(Error::from)?;
let securable = SecurableRef::Table(table_id, operation);
self.build_store(credential, securable).await
}
async fn table_storage_location(&self, full_name: &str) -> Result<Option<String>> {
let table = self
.uc
.tables_client()
.get_table(&GetTableRequest {
full_name: full_name.to_string(),
include_browse: Some(false),
include_delta_metadata: Some(false),
include_manifest_capabilities: Some(false),
..Default::default()
})
.await
.map_err(Error::from)?;
Ok(table.storage_location.filter(|s| !s.is_empty()))
}
pub async fn for_volume(
&self,
volume: impl Into<VolumeReference>,
operation: VolumeOperation,
) -> Result<UCStore> {
let volume = volume.into();
if let VolumeReference::Name(name) = &volume
&& let Some(location) = self.volume_storage_location(name).await?
&& let Ok(url) = Url::parse(&location)
{
if url.scheme() == "file" {
let path_op = match operation {
VolumeOperation::Read => PathOperation::Read,
VolumeOperation::ReadWrite => PathOperation::ReadWrite,
};
return local_store(&url, path_op);
}
#[cfg(target_arch = "wasm32")]
if matches!(self.storage_access, proxy::StorageAccess::Proxy { .. }) {
let seg = format!("vol:{name}");
return self.proxy_store(&seg, &url);
}
}
let (credential, volume_id) = self
.creds
.temporary_volume_credential(volume, operation)
.await
.map_err(Error::from)?;
let securable = SecurableRef::Volume(volume_id, operation);
self.build_store(credential, securable).await
}
async fn volume_storage_location(&self, name: &str) -> Result<Option<String>> {
let volume = self
.uc
.volumes_client()
.get_volume(&GetVolumeRequest {
name: name.to_string(),
include_browse: Some(false),
..Default::default()
})
.await
.map_err(Error::from)?;
Ok(Some(volume.storage_location).filter(|s| !s.is_empty()))
}
pub async fn for_path(&self, path: &Url, operation: PathOperation) -> Result<UCStore> {
if path.scheme() == "file" {
return local_store(path, operation);
}
#[cfg(target_arch = "wasm32")]
if matches!(self.storage_access, proxy::StorageAccess::Proxy { .. }) {
let seg = proxy::path_securable(path);
return self.proxy_store(&seg, path);
}
let (credential, _resolved) = self
.creds
.temporary_path_credential(path.clone(), operation, false)
.await
.map_err(Error::from)?;
let securable = SecurableRef::Path(path.clone(), operation, Some(false));
self.build_store(credential, securable).await
}
pub async fn dry_run_path(&self, path: &Url, operation: PathOperation) -> Result<UCStore> {
if path.scheme() == "file" {
return local_store(path, operation);
}
let (credential, _resolved) = self
.creds
.temporary_path_credential(path.clone(), operation, true)
.await
.map_err(Error::from)?;
let securable = SecurableRef::Path(path.clone(), operation, Some(true));
self.build_store(credential, securable).await
}
#[cfg(target_arch = "wasm32")]
fn proxy_store(&self, securable_seg: &str, location: &Url) -> Result<UCStore> {
let proxy::StorageAccess::Proxy { base } = &self.storage_access else {
return Err(Error::invalid_config(
"proxy_store called without a proxy posture".to_string(),
)
.into());
};
let path = match parse_azurite(location) {
Some(loc) => Path::from(loc.prefix),
None => Path::from_url_path(location.path())?,
};
let root = proxy::build_proxy_store(base, securable_seg, path.clone(), self.token())?;
Ok(UCStore {
root,
url: location.clone(),
path,
})
}
#[cfg(target_arch = "wasm32")]
fn token(&self) -> Option<&str> {
self.token.as_deref()
}
async fn build_store(
&self,
credential: TemporaryCredential,
securable: SecurableRef,
) -> Result<UCStore> {
let url = Url::parse(&credential.url).map_err(Error::from)?;
let path = match parse_azurite(&url) {
Some(loc) => Path::from(loc.prefix),
None => Path::from_url_path(url.path())?,
};
let store = self.to_store(credential, securable).await?;
Ok(UCStore {
root: store,
url,
path,
})
}
async fn to_store(
&self,
credential: TemporaryCredential,
securable: SecurableRef,
) -> Result<Arc<dyn ObjectStore>> {
if as_azure(&credential).is_ok() {
let url = Url::parse(&credential.url).map_err(Error::from)?;
if let Some(loc) = parse_azurite(&url) {
let sas = azure_sas_token(&credential).ok_or_else(|| {
Error::invalid_config(
"Azurite store requires a SAS-token credential".to_string(),
)
})?;
#[allow(unused_mut)]
let mut builder = MicrosoftAzureBuilder::new()
.with_use_emulator(true)
.with_account(loc.account)
.with_container_name(loc.container)
.with_config(object_store::azure::AzureConfigKey::SasKey, sas);
#[cfg(not(target_arch = "wasm32"))]
if let Some(handle) = &self.io_handle {
builder =
builder.with_http_connector(SpawnedReqwestConnector::new(handle.clone()));
}
#[cfg(target_arch = "wasm32")]
{
builder = builder.with_retry(object_store::RetryConfig {
max_retries: 0,
..Default::default()
});
}
let store = builder.build()?;
return Ok(Arc::new(store));
}
let provider = new_azure(self.creds.clone(), &credential, securable).await?;
#[allow(unused_mut)]
let mut builder = MicrosoftAzureBuilder::new()
.with_url(url.to_string())
.with_credentials(Arc::new(provider));
#[cfg(not(target_arch = "wasm32"))]
if let Some(handle) = &self.io_handle {
builder = builder.with_http_connector(SpawnedReqwestConnector::new(handle.clone()));
}
#[cfg(target_arch = "wasm32")]
{
builder = builder.with_retry(object_store::RetryConfig {
max_retries: 0,
..Default::default()
});
}
let store = builder.build()?;
return Ok(Arc::new(store));
}
#[cfg(not(target_arch = "wasm32"))]
if as_aws(&credential).is_ok() {
let access_point = aws_access_point(&credential);
let provider = new_aws(self.creds.clone(), &credential, securable).await?;
let url = Url::parse(&credential.url).map_err(Error::from)?;
let mut builder = AmazonS3Builder::new()
.with_url(url.to_string())
.with_credentials(Arc::new(provider));
if let Some(region) = self
.aws_region
.clone()
.or_else(|| std::env::var("AWS_REGION").ok())
{
builder = builder.with_region(region);
}
if let Some(ap) = access_point {
builder = builder.with_bucket_name(ap);
}
if let Some(handle) = &self.io_handle {
builder = builder.with_http_connector(SpawnedReqwestConnector::new(handle.clone()));
}
let store = builder.build()?;
return Ok(Arc::new(store));
}
#[cfg(not(target_arch = "wasm32"))]
if as_gcp(&credential).is_ok() {
let provider = new_gcp(self.creds.clone(), &credential, securable).await?;
let url = Url::parse(&credential.url).map_err(Error::from)?;
let mut builder = GoogleCloudStorageBuilder::new()
.with_url(url.to_string())
.with_credentials(Arc::new(provider));
if let Some(handle) = &self.io_handle {
builder = builder.with_http_connector(SpawnedReqwestConnector::new(handle.clone()));
}
let store = builder.build()?;
return Ok(Arc::new(store));
}
#[cfg(target_arch = "wasm32")]
if as_aws(&credential).is_ok() || as_gcp(&credential).is_ok() {
return Err(Error::invalid_config(
"AWS/GCP object stores are not supported on wasm (Azure-first)",
)
.into());
}
Err(
Error::InvalidCredential("Failed to match credential with storage type".to_string())
.into(),
)
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Operation {
Read,
ReadWrite,
}
impl Operation {
fn into_volume(self) -> VolumeOperation {
match self {
Operation::Read => VolumeOperation::Read,
Operation::ReadWrite => VolumeOperation::ReadWrite,
}
}
fn into_table(self) -> TableOperation {
match self {
Operation::Read => TableOperation::Read,
Operation::ReadWrite => TableOperation::ReadWrite,
}
}
fn into_path(self) -> PathOperation {
match self {
Operation::Read => PathOperation::Read,
Operation::ReadWrite => PathOperation::ReadWrite,
}
}
}
impl From<Operation> for TableOperation {
fn from(op: Operation) -> Self {
op.into_table()
}
}
impl From<Operation> for VolumeOperation {
fn from(op: Operation) -> Self {
op.into_volume()
}
}
impl From<Operation> for PathOperation {
fn from(op: Operation) -> Self {
op.into_path()
}
}
#[cfg(not(target_arch = "wasm32"))]
fn local_store(url: &Url, operation: PathOperation) -> Result<UCStore> {
if cfg!(windows) {
return Err(Error::invalid_config(format!(
"local (file://) storage is not supported on Windows: {url}"
))
.into());
}
let dir = url
.to_file_path()
.map_err(|_| Error::invalid_url(format!("not a valid local file path: {url}")))?;
if matches!(
operation,
PathOperation::ReadWrite | PathOperation::CreateTable
) {
std::fs::create_dir_all(&dir).map_err(|e| {
Error::invalid_config(format!("failed to create local directory {dir:?}: {e}"))
})?;
}
let store = LocalFileSystem::new();
let path = Path::from_url_path(url.path())?;
Ok(UCStore {
root: Arc::new(store),
url: url.clone(),
path,
})
}
#[cfg(not(target_arch = "wasm32"))]
#[derive(Debug)]
struct StaticArcProvider<T>(Arc<T>);
#[cfg(not(target_arch = "wasm32"))]
#[async_trait::async_trait]
impl<T: std::fmt::Debug + Send + Sync> object_store::CredentialProvider for StaticArcProvider<T> {
type Credential = T;
async fn get_credential(&self) -> Result<Arc<T>> {
Ok(self.0.clone())
}
}
#[cfg(not(target_arch = "wasm32"))]
pub fn store_from_vended_credential(
credential: &TemporaryCredential,
io_handle: Option<tokio::runtime::Handle>,
aws_region: Option<String>,
) -> Result<UCStore> {
let url = Url::parse(&credential.url).map_err(Error::from)?;
if url.scheme() == "file" {
let store = LocalFileSystem::new();
let path = Path::from_url_path(url.path())?;
return Ok(UCStore {
root: Arc::new(store),
url,
path,
});
}
let path = match parse_azurite(&url) {
Some(loc) => Path::from(loc.prefix),
None => Path::from_url_path(url.path())?,
};
let root: Arc<dyn ObjectStore> = if as_azure(credential).is_ok() {
if let Some(loc) = parse_azurite(&url) {
let sas = azure_sas_token(credential).ok_or_else(|| {
Error::invalid_config("Azurite store requires a SAS-token credential".to_string())
})?;
let mut builder = MicrosoftAzureBuilder::new()
.with_use_emulator(true)
.with_account(loc.account)
.with_container_name(loc.container)
.with_config(object_store::azure::AzureConfigKey::SasKey, sas);
if let Some(handle) = &io_handle {
builder = builder.with_http_connector(SpawnedReqwestConnector::new(handle.clone()));
}
Arc::new(builder.build()?)
} else {
let token = as_azure(credential)?.token;
let mut builder = MicrosoftAzureBuilder::new()
.with_url(url.to_string())
.with_credentials(Arc::new(StaticArcProvider(token)));
if let Some(handle) = &io_handle {
builder = builder.with_http_connector(SpawnedReqwestConnector::new(handle.clone()));
}
Arc::new(builder.build()?)
}
} else if as_aws(credential).is_ok() {
let access_point = aws_access_point(credential);
let token = as_aws(credential)?.token;
let mut builder = AmazonS3Builder::new()
.with_url(url.to_string())
.with_credentials(Arc::new(StaticArcProvider(token)));
if let Some(region) = aws_region.or_else(|| std::env::var("AWS_REGION").ok()) {
builder = builder.with_region(region);
}
if let Some(ap) = access_point {
builder = builder.with_bucket_name(ap);
}
if let Some(handle) = &io_handle {
builder = builder.with_http_connector(SpawnedReqwestConnector::new(handle.clone()));
}
Arc::new(builder.build()?)
} else if as_gcp(credential).is_ok() {
let token = as_gcp(credential)?.token;
let mut builder = GoogleCloudStorageBuilder::new()
.with_url(url.to_string())
.with_credentials(Arc::new(StaticArcProvider(token)));
if let Some(handle) = &io_handle {
builder = builder.with_http_connector(SpawnedReqwestConnector::new(handle.clone()));
}
Arc::new(builder.build()?)
} else {
return Err(Error::InvalidCredential(
"Failed to match credential with storage type".to_string(),
)
.into());
};
Ok(UCStore { root, url, path })
}
#[cfg(target_arch = "wasm32")]
fn local_store(url: &Url, _operation: PathOperation) -> Result<UCStore> {
Err(Error::invalid_config(format!(
"local (file://) storage is not supported on wasm: {url}"
))
.into())
}
struct AzuriteLocation {
account: String,
container: String,
prefix: String,
}
fn parse_azurite(url: &Url) -> Option<AzuriteLocation> {
const EMULATOR_ACCOUNT: &str = "devstoreaccount1";
if url.scheme() == "azurite" {
let container = url.host_str().filter(|s| !s.is_empty())?.to_owned();
let prefix = url.path().trim_start_matches('/').to_owned();
return Some(AzuriteLocation {
account: EMULATOR_ACCOUNT.to_owned(),
container,
prefix,
});
}
let is_localhost = matches!(url.host_str(), Some("localhost") | Some("127.0.0.1"));
if url.scheme() == "http" && is_localhost && url.port() == Some(10000) {
let mut segments = url.path_segments()?;
let account = segments.next().filter(|s| !s.is_empty())?.to_owned();
let container = segments.next().filter(|s| !s.is_empty())?.to_owned();
let prefix = segments.collect::<Vec<_>>().join("/");
return Some(AzuriteLocation {
account,
container,
prefix,
});
}
None
}
fn azure_sas_token(credential: &TemporaryCredential) -> Option<String> {
use unitycatalog_common::temporary_credentials::v1::temporary_credential::Credentials;
match credential.credentials.as_ref()? {
Credentials::AzureUserDelegationSas(sas) => Some(sas.sas_token.clone()),
_ => None,
}
}
fn extend_prefix(store: UCStore, extra: &str) -> UCStore {
let mut url = store.url.clone();
{
let mut segs = url.path_segments_mut().expect("cloud URL has a path");
segs.pop_if_empty();
for part in extra.split('/').filter(|p| !p.is_empty()) {
segs.push(part);
}
}
let new_path = if store.path.as_ref().is_empty() {
Path::from(extra)
} else {
let base = store.path.as_ref().trim_end_matches('/');
let extra = extra.trim_start_matches('/');
Path::from(format!("{base}/{extra}"))
};
UCStore {
root: store.root,
url,
path: new_path,
}
}
#[cfg(all(test, not(target_arch = "wasm32")))]
mod tests {
use futures::TryStreamExt;
#[cfg(not(windows))]
use object_store::{ObjectStoreExt, PutPayload};
use super::*;
async fn offline_factory() -> UnityObjectStoreFactory {
UnityObjectStoreFactory::builder()
.with_uri("http://127.0.0.1:0/api/2.1/unity-catalog/")
.with_allow_unauthenticated(true)
.build()
.await
.unwrap()
}
#[cfg(not(windows))]
#[tokio::test]
async fn for_url_file_roundtrips_without_vending() {
let dir = tempfile::tempdir().unwrap();
let url = Url::from_directory_path(dir.path()).unwrap();
let factory = offline_factory().await;
let store = factory
.for_url(url.as_str(), Operation::ReadWrite)
.await
.unwrap();
let dyn_store = store.as_dyn();
let path = object_store::path::Path::from("hello.txt");
dyn_store
.put(&path, PutPayload::from_static(b"world"))
.await
.unwrap();
let listing: Vec<_> = dyn_store.list(None).try_collect().await.unwrap();
assert_eq!(listing.len(), 1, "expected exactly one object");
assert_eq!(listing[0].location, path);
let got = dyn_store.get(&path).await.unwrap().bytes().await.unwrap();
assert_eq!(&got[..], b"world");
}
#[cfg(not(windows))]
#[tokio::test]
async fn for_path_file_skips_vending() {
let dir = tempfile::tempdir().unwrap();
let url = Url::from_directory_path(dir.path()).unwrap();
let factory = offline_factory().await;
let store = factory.for_path(&url, PathOperation::Read).await.unwrap();
let listing: Vec<_> = store.as_dyn().list(None).try_collect().await.unwrap();
assert!(listing.is_empty());
assert_eq!(store.url(), &url);
}
#[cfg(not(windows))]
#[tokio::test]
async fn local_store_read_write_creates_dir() {
let dir = tempfile::tempdir().unwrap();
let missing = dir.path().join("not-yet-here");
let url = Url::from_directory_path(&missing).unwrap();
let store = local_store(&url, PathOperation::ReadWrite).unwrap();
assert!(missing.exists(), "ReadWrite must create the root directory");
let path = object_store::path::Path::from("a.bin");
store
.as_dyn()
.put(&path, PutPayload::from_static(b"x"))
.await
.unwrap();
assert!(missing.join("a.bin").exists());
}
#[cfg(not(windows))]
#[tokio::test]
async fn local_store_root_resolves_full_path() {
let dir = tempfile::tempdir().unwrap();
let table_dir = dir.path().join("mytable");
let url = Url::from_directory_path(&table_dir).unwrap();
let store = local_store(&url, PathOperation::ReadWrite).unwrap();
let full = Path::from(format!("{}/part-0.parquet", store.prefix()));
store
.root()
.put(&full, PutPayload::from_static(b"data"))
.await
.unwrap();
let got = store
.root()
.get(&full)
.await
.unwrap()
.bytes()
.await
.unwrap();
assert_eq!(&got[..], b"data");
let entries: Vec<_> = std::fs::read_dir(&table_dir)
.unwrap()
.map(|e| e.unwrap().file_name())
.collect();
assert_eq!(entries, vec![std::ffi::OsString::from("part-0.parquet")]);
}
#[cfg(not(windows))]
#[test]
fn local_store_rejects_non_file_url() {
let url = Url::parse("s3://bucket/prefix/").unwrap();
match local_store(&url, PathOperation::Read) {
Ok(_) => panic!("expected an error for a non-file URL"),
Err(e) => assert!(
e.to_string().contains("not a valid local file path"),
"unexpected error: {e}"
),
}
}
#[test]
fn build_with_io_runtime_carries_handle() {
let io_runtime = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.unwrap();
let handle = io_runtime.handle().clone();
let driver = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.unwrap();
driver.block_on(async {
let factory = UnityObjectStoreFactory::builder()
.with_uri("http://localhost:8080/api/2.1/unity-catalog/")
.with_allow_unauthenticated(true)
.with_io_runtime(handle.clone())
.build()
.await
.unwrap();
assert!(factory.io_handle.is_some());
let factory = UnityObjectStoreFactory::builder()
.with_uri("http://localhost:8080/api/2.1/unity-catalog/")
.with_allow_unauthenticated(true)
.with_io_runtime(handle.clone())
.with_io_runtime(None)
.build()
.await
.unwrap();
assert!(factory.io_handle.is_none());
});
}
#[test]
#[ignore]
fn list_store_via_temp_credential_on_io_runtime() {
let databricks_host = std::env::var("DATABRICKS_HOST").unwrap();
let databricks_token = std::env::var("DATABRICKS_TOKEN").unwrap();
let io_runtime = std::thread::spawn(|| {
tokio::runtime::Builder::new_multi_thread()
.enable_all()
.build()
.unwrap()
})
.join()
.unwrap();
let io_handle = io_runtime.handle().clone();
let main_runtime = tokio::runtime::Builder::new_current_thread()
.build()
.unwrap();
main_runtime.block_on(async move {
let factory = UnityObjectStoreFactory::builder()
.with_uri(format!("{databricks_host}/api/2.1/unity-catalog/"))
.with_token(databricks_token)
.with_aws_region("eu-north-1".to_string())
.with_io_runtime(io_handle)
.build()
.await
.unwrap();
let volume_path = url::Url::parse("s3://open-lakehouse-dev/volumes/").unwrap();
let store = factory
.for_path(&volume_path, PathOperation::Read)
.await
.unwrap();
let files: Vec<_> = store.as_dyn().list(None).try_collect().await.unwrap();
println!("files: {files:?}");
});
}
#[test]
fn parse_azurite_path_style() {
let url =
Url::parse("http://127.0.0.1:10000/devstoreaccount1/mycontainer/some/prefix").unwrap();
let loc = parse_azurite(&url).expect("should parse path-style azurite URL");
assert_eq!(loc.account, "devstoreaccount1");
assert_eq!(loc.container, "mycontainer");
assert_eq!(loc.prefix, "some/prefix");
}
#[test]
fn parse_azurite_custom_scheme() {
let url = Url::parse("azurite://mycontainer/some/prefix").unwrap();
let loc = parse_azurite(&url).expect("should parse azurite:// URL");
assert_eq!(loc.account, "devstoreaccount1");
assert_eq!(loc.container, "mycontainer");
assert_eq!(loc.prefix, "some/prefix");
}
#[test]
fn parse_azurite_localhost_alias() {
let url = Url::parse("http://localhost:10000/acct/cont/p").unwrap();
let loc = parse_azurite(&url).expect("localhost is also an azurite host");
assert_eq!(loc.account, "acct");
assert_eq!(loc.container, "cont");
assert_eq!(loc.prefix, "p");
}
#[test]
fn parse_azurite_rejects_real_cloud_urls() {
for raw in [
"https://acct.blob.core.windows.net/container/path",
"s3://bucket/prefix",
"http://127.0.0.1:9000/acct/cont/p", ] {
let url = Url::parse(raw).unwrap();
assert!(parse_azurite(&url).is_none(), "should not match: {raw}");
}
}
#[tokio::test]
async fn azurite_store_targets_emulator_and_roots_prefix() {
use unitycatalog_common::temporary_credentials::v1::AzureUserDelegationSas;
let url = "http://127.0.0.1:10000/devstoreaccount1/mycontainer/tbl/data";
let credential = TemporaryCredential {
expiration_time: now_epoch_millis() + 3_600_000,
url: url.to_string(),
credentials: Some(
AzureUserDelegationSas {
sas_token: "sv=2021-08-06&ss=b&srt=co&sp=rl&se=2999-01-01T00:00:00Z&sig=AAAA"
.to_string(),
..Default::default()
}
.into(),
),
..Default::default()
};
let factory = offline_factory().await;
let securable =
SecurableRef::Path(Url::parse(url).unwrap(), PathOperation::Read, Some(false));
let store = factory.build_store(credential, securable).await.unwrap();
assert_eq!(store.prefix(), &Path::from("tbl/data"));
}
fn now_epoch_millis() -> i64 {
use std::time::{SystemTime, UNIX_EPOCH};
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap()
.as_millis() as i64
}
fn azurite_credential_body() -> String {
format!(
r#"{{"expiration_time":{},"url":"http://127.0.0.1:10000/devstoreaccount1/mycontainer/prefix/","azure_user_delegation_sas":{{"sas_token":"sv=2021-08-06&ss=b&srt=co&sp=rl&se=2999-01-01T00:00:00Z&sig=AAAA"}}}}"#,
now_epoch_millis() + 3_600_000
)
}
async fn factory_at(base_url: &str) -> UnityObjectStoreFactory {
UnityObjectStoreFactory::builder()
.with_uri(format!("{base_url}/api/2.1/unity-catalog/"))
.with_token("proxy-service-token".to_string())
.build()
.await
.unwrap()
}
#[tokio::test]
async fn forwarded_user_injects_header_and_drops_token() {
let mut server = mockito::Server::new_async().await;
let mock = server
.mock("POST", "/api/2.1/unity-catalog/temporary-path-credentials")
.match_header("x-forwarded-user", "alice")
.match_header("authorization", mockito::Matcher::Missing)
.with_status(200)
.with_header("content-type", "application/json")
.with_body(azurite_credential_body())
.expect(1)
.create_async()
.await;
let factory = factory_at(&server.url()).await;
let scoped = factory
.with_forwarded_user("x-forwarded-user", Some("alice"))
.unwrap();
let url = Url::parse("abfss://mycontainer@devstoreaccount1/prefix/").unwrap();
scoped
.for_path(&url, PathOperation::Read)
.await
.expect("vend + store build should succeed against the mock");
mock.assert_async().await;
}
#[tokio::test]
async fn anonymous_sends_no_forwarded_header_and_keeps_token() {
let mut server = mockito::Server::new_async().await;
let mock = server
.mock("POST", "/api/2.1/unity-catalog/temporary-path-credentials")
.match_header("x-forwarded-user", mockito::Matcher::Missing)
.match_header("authorization", "Bearer proxy-service-token")
.with_status(200)
.with_header("content-type", "application/json")
.with_body(azurite_credential_body())
.expect(1)
.create_async()
.await;
let factory = factory_at(&server.url()).await;
let scoped = factory
.with_forwarded_user("x-forwarded-user", None)
.unwrap();
let url = Url::parse("abfss://mycontainer@devstoreaccount1/prefix/").unwrap();
scoped
.for_path(&url, PathOperation::Read)
.await
.expect("vend + store build should succeed against the mock");
mock.assert_async().await;
}
#[tokio::test]
async fn forwarded_user_honors_custom_header_name() {
let mut server = mockito::Server::new_async().await;
let mock = server
.mock("POST", "/api/2.1/unity-catalog/temporary-path-credentials")
.match_header("x-remote-user", "bob")
.with_status(200)
.with_header("content-type", "application/json")
.with_body(azurite_credential_body())
.expect(1)
.create_async()
.await;
let factory = factory_at(&server.url()).await;
let scoped = factory
.with_forwarded_user("x-remote-user", Some("bob"))
.unwrap();
let url = Url::parse("abfss://mycontainer@devstoreaccount1/prefix/").unwrap();
scoped.for_path(&url, PathOperation::Read).await.unwrap();
mock.assert_async().await;
}
#[tokio::test]
async fn forwarded_user_rejects_invalid_header() {
let factory = factory_at("http://127.0.0.1:0").await;
assert!(
factory
.with_forwarded_user("bad header", Some("alice"))
.is_err()
);
assert!(
factory
.with_forwarded_user("x-forwarded-user", Some("a\nb"))
.is_err()
);
}
#[tokio::test]
async fn forwarded_header_service_inserts_and_delegates() {
use olai_http::service::{HttpService, ReqwestService};
use reqwest::header::{HeaderName, HeaderValue};
let mut server = mockito::Server::new_async().await;
let mock = server
.mock("GET", "/echo")
.match_header("x-forwarded-user", "carol")
.with_status(204)
.expect(1)
.create_async()
.await;
let client = reqwest::Client::new();
let svc = ForwardedHeaderService {
inner: Arc::new(ReqwestService::new(client.clone())),
name: HeaderName::from_static("x-forwarded-user"),
value: HeaderValue::from_static("carol"),
};
let mut request = client
.get(format!("{}/echo", server.url()))
.build()
.unwrap();
request
.headers_mut()
.insert("x-forwarded-user", HeaderValue::from_static("stale"));
let resp = svc.call(request).await.unwrap();
assert_eq!(resp.status(), 204);
mock.assert_async().await;
}
}