use std::path::{Path, PathBuf};
use std::sync::Arc;
use super::backend::Backend;
use super::backends::LocalBackend;
use super::engine::EngineFactory;
use super::errors::{err, Category, Result};
use super::keys::validate_object_id;
use super::object::{open_object, DurableObject, OpenOptions, Tuning};
use super::ChdbEngine;
pub type BackendFactory = Box<dyn Fn(&str) -> Result<Arc<dyn Backend>> + Send + Sync>;
pub struct Namespace {
location: String,
backend_factory: BackendFactory,
engine_factory: EngineFactory,
owner: Option<String>,
scratch_root: Option<PathBuf>,
tuning: Tuning,
}
impl std::fmt::Debug for Namespace {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("Namespace")
.field("location", &self.location)
.field("owner", &self.owner)
.finish_non_exhaustive()
}
}
#[cfg(feature = "durable-s3")]
fn query_of(url: &str) -> Vec<(String, String)> {
let Some((_, query)) = url.split_once('?') else {
return Vec::new();
};
query
.split('&')
.filter(|pair| !pair.is_empty())
.map(|pair| {
let (name, value) = pair.split_once('=').unwrap_or((pair, ""));
(percent_decode(name), percent_decode(value))
})
.collect()
}
#[cfg(feature = "durable-s3")]
fn percent_decode(value: &str) -> String {
let bytes = value.as_bytes();
let mut out = Vec::with_capacity(bytes.len());
let mut index = 0;
while index < bytes.len() {
match bytes[index] {
b'%' if index + 2 < bytes.len() => {
let hex = std::str::from_utf8(&bytes[index + 1..index + 3]).unwrap_or("");
match u8::from_str_radix(hex, 16) {
Ok(byte) => {
out.push(byte);
index += 3;
}
Err(_) => {
out.push(b'%');
index += 1;
}
}
}
byte => {
out.push(byte);
index += 1;
}
}
}
String::from_utf8_lossy(&out).into_owned()
}
#[cfg(feature = "durable-s3")]
fn s3_location(url: &str) -> Result<(String, String)> {
let location = url
.trim_start_matches("s3://")
.split('?')
.next()
.unwrap_or_default();
let (bucket, prefix) = match location.split_once('/') {
Some((bucket, prefix)) => (bucket, prefix.trim_matches('/')),
None => (location, ""),
};
if bucket.is_empty() {
return Err(err(
Category::Backend,
format!("durable: an s3 namespace URL needs a bucket, got {url:?}"),
));
}
Ok((bucket.to_string(), prefix.to_string()))
}
fn local_root(url: &str) -> Result<PathBuf> {
let url = url.split('?').next().unwrap_or(url);
let path = url
.strip_prefix("file://")
.or_else(|| url.strip_prefix("file:"))
.or_else(|| url.strip_prefix("local://"))
.or_else(|| url.strip_prefix("local:"))
.unwrap_or(url);
if let Some(scheme_end) = path.find("://") {
return Err(err(
Category::Backend,
format!(
"durable: this build has no backend for the {:?} scheme; pass a backend factory \
with Namespace::with_backend for anything but a local directory",
&path[..scheme_end]
),
));
}
let path = Path::new(path);
if !path.is_absolute() {
return Err(err(
Category::Backend,
format!("durable: a local namespace needs an absolute path, got {url:?}"),
));
}
Ok(path.to_path_buf())
}
impl Namespace {
pub fn new(url: &str) -> Result<Self> {
if url.starts_with("s3://") {
return Self::s3(url);
}
let root = local_root(url)?;
Ok(Self::with_backend(Box::new(move |id| {
Ok(Arc::new(LocalBackend::new(root.join(id))?))
}))
.named(url))
}
#[cfg(feature = "durable-s3")]
fn s3(url: &str) -> Result<Self> {
use super::backends::{S3Backend, S3Options};
let (bucket, base_prefix) = s3_location(url)?;
let query = query_of(url);
let value = |name: &str| {
query
.iter()
.find(|(key, _)| key == name)
.map(|(_, value)| value.clone())
.filter(|value| !value.is_empty())
};
let path_style = value("path_style")
.or_else(|| value("pathStyle"))
.map(|raw| raw == "true" || raw == "1");
let region = value("region");
let endpoint = value("endpoint");
Ok(Self::with_backend(Box::new(move |id| {
let prefix = if base_prefix.is_empty() {
id.to_string()
} else {
format!("{base_prefix}/{id}")
};
Ok(Arc::new(S3Backend::new(S3Options {
bucket: bucket.clone(),
prefix,
region: region.clone(),
endpoint: endpoint.clone(),
path_style,
..S3Options::default()
})?))
}))
.named(url))
}
#[cfg(not(feature = "durable-s3"))]
fn s3(url: &str) -> Result<Self> {
Err(err(
Category::Backend,
format!(
"durable: {url:?} needs the `durable-s3` feature, which is off in this build; enable it, or pass your own backend to Namespace::with_backend"
),
))
}
pub fn with_backend(backend_factory: BackendFactory) -> Self {
Self {
location: "<custom backend>".to_string(),
backend_factory,
engine_factory: Box::new(|| Ok(Box::new(ChdbEngine::new()))),
owner: None,
scratch_root: None,
tuning: Tuning::default(),
}
}
fn named(mut self, location: &str) -> Self {
self.location = location.to_string();
self
}
pub fn with_engine(mut self, engine_factory: EngineFactory) -> Self {
self.engine_factory = engine_factory;
self
}
pub fn with_owner(mut self, owner: impl Into<String>) -> Self {
self.owner = Some(owner.into());
self
}
pub fn with_scratch_root(mut self, root: impl Into<PathBuf>) -> Self {
self.scratch_root = Some(root.into());
self
}
pub fn with_tuning(mut self, tuning: Tuning) -> Self {
self.tuning = tuning;
self
}
pub fn location(&self) -> &str {
&self.location
}
pub fn open(&self, id: &str, options: OpenOptions) -> Result<(DurableObject, bool)> {
validate_object_id(id)?;
let backend = (self.backend_factory)(id)?;
let options = OpenOptions {
owner: options.owner.or_else(|| self.owner.clone()),
scratch_root: options.scratch_root.or_else(|| self.scratch_root.clone()),
tuning: if options.tuning == Tuning::default() {
self.tuning.clone()
} else {
options.tuning
},
..options
};
open_object(id, backend, &self.engine_factory, options)
}
pub fn scan<I, S>(
&self,
sql: &str,
ids: I,
format: crate::format::OutputFormat,
) -> Result<Vec<(String, Vec<u8>)>>
where
I: IntoIterator<Item = S>,
S: AsRef<str>,
{
let mut results = Vec::new();
for id in ids {
let id = id.as_ref();
let (object, _) = self.open(
id,
OpenOptions {
read_only: true,
..OpenOptions::default()
},
)?;
let query = object.query(sql, format);
let close = object.close();
match (query, close) {
(Ok(bytes), Ok(())) => results.push((id.to_owned(), bytes)),
(Err(error), _) => return Err(error),
(Ok(_), Err(error)) => return Err(error),
}
}
Ok(results)
}
}
#[cfg(test)]
mod test {
use super::*;
#[test]
fn a_local_url_names_a_directory_in_three_spellings() -> Result<()> {
assert_eq!(local_root("file:///var/lib/x")?, Path::new("/var/lib/x"));
assert_eq!(local_root("local:/var/lib/x")?, Path::new("/var/lib/x"));
assert_eq!(local_root("/var/lib/x")?, Path::new("/var/lib/x"));
Ok(())
}
#[test]
fn a_scheme_this_build_cannot_serve_fails_at_configuration_time() {
let error = local_root("gs://bucket/prefix").expect_err("no GCS backend here");
assert_eq!(error.category(), Category::Backend);
assert!(error.to_string().contains("backend factory"), "{error}");
}
#[cfg(feature = "durable-s3")]
#[test]
fn an_s3_url_splits_into_a_bucket_and_a_prefix() {
assert_eq!(
s3_location("s3://my-bucket/durable/objects?region=eu-central-1").unwrap(),
("my-bucket".to_string(), "durable/objects".to_string())
);
assert_eq!(
s3_location("s3://my-bucket").unwrap(),
("my-bucket".to_string(), String::new()),
"a bucket with no prefix is the bucket root"
);
assert_eq!(
s3_location("s3://my-bucket/").unwrap(),
("my-bucket".to_string(), String::new())
);
assert_eq!(
s3_location("s3://").unwrap_err().category(),
Category::Backend
);
}
#[cfg(feature = "durable-s3")]
#[test]
fn a_query_value_survives_being_escaped_by_a_url_builder() {
let query = query_of("s3://b/p?region=auto&endpoint=https%3A%2F%2Fx.example&pathStyle=1");
assert_eq!(query[0], ("region".to_string(), "auto".to_string()));
assert_eq!(
query[1],
("endpoint".to_string(), "https://x.example".to_string()),
"an escaped endpoint has to mean the same as an unescaped one"
);
assert_eq!(query[2], ("pathStyle".to_string(), "1".to_string()));
assert!(query_of("s3://b/p").is_empty());
assert_eq!(percent_decode("100%"), "100%");
assert_eq!(percent_decode("a%2Fb"), "a/b");
}
#[test]
fn a_relative_root_is_refused() {
assert!(local_root("file://relative/path").is_err());
assert!(local_root("chdb-durable").is_err());
}
#[test]
fn an_object_id_cannot_address_another_objects_prefix() {
let namespace = Namespace::new("/tmp/chdb-durable-test").expect("a local namespace");
for id in ["", "..", "a/b"] {
let error = namespace
.open(id, OpenOptions::default())
.expect_err("an id that is not one segment");
assert_eq!(error.category(), Category::Backend, "{id:?}");
}
}
}