use std::collections::HashMap;
use std::env;
use std::path::PathBuf;
use std::str::FromStr as _;
use martin_core::tiles::BoxedSource;
use martin_core::tiles::pmtiles::{PmtCache, PmtCacheInstance, PmtilesSource};
use serde::{Deserialize, Serialize};
use tracing::{trace, warn};
use url::Url;
use crate::MartinResult;
use crate::config::file::{
ConfigFileError, ConfigFileResult, ConfigurationLivecycleHooks, TileSourceConfiguration,
UnrecognizedKeys, UnrecognizedValues,
};
#[serde_with::skip_serializing_none]
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct PmtConfig {
pub directory_cache_size_mb: Option<u64>,
#[serde(skip)]
pub options: HashMap<String, String>,
#[serde(flatten, skip_serializing)]
pub unrecognized: UnrecognizedValues,
#[serde(skip)]
pub pmtiles_directory_cache: PmtCache,
}
impl PartialEq for PmtConfig {
fn eq(&self, other: &Self) -> bool {
self.options == other.options && self.unrecognized == other.unrecognized
}
}
impl ConfigurationLivecycleHooks for PmtConfig {
fn finalize(&mut self) -> ConfigFileResult<()> {
self.partition_options_and_unrecognized();
self.migrate_deprecated_keys();
Ok(())
}
fn get_unrecognized_keys(&self) -> UnrecognizedKeys {
self.unrecognized.keys().cloned().collect()
}
}
impl PmtConfig {
fn partition_options_and_unrecognized(&mut self) {
for (key, value) in self.unrecognized.clone() {
let key_could_configure_object_store =
object_store::aws::AmazonS3ConfigKey::from_str(key.as_str()).is_ok()
|| object_store::gcp::GoogleConfigKey::from_str(key.as_str()).is_ok()
|| object_store::azure::AzureConfigKey::from_str(key.as_str()).is_ok()
|| object_store::client::ClientConfigKey::from_str(key.as_str()).is_ok();
if key_could_configure_object_store {
self.unrecognized
.remove(&key)
.expect("key should exist in the hashmap");
let _ = match value {
serde_yaml::Value::Bool(b) => self.options.insert(key.clone(), b.to_string()),
serde_yaml::Value::Number(n) => self.options.insert(key.clone(), n.to_string()),
serde_yaml::Value::String(s) => self.options.insert(key.clone(), s.clone()),
v => {
warn!(
"Ignoring unrecognized configuration key 'pmtiles.{key}': {v:?}. Only boolean, string or number values are allowed here. Please check your configuration file for typos."
);
None
}
};
}
}
}
fn migrate_deprecated_keys(&mut self) {
if self.unrecognized.contains_key("dir_cache_size_mb") {
warn!(
"dir_cache_size_mb is no longer used. Instead, use cache_size_mb param in the root of the config file."
);
}
if !self.options.contains_key("allow_http") {
warn!(
"Defaulting `pmtiles.allow_http` to `true`. This is likely to become an error in the future for better security."
);
self.options
.insert("allow_http".to_string(), true.to_string());
}
for key in ["aws_s3_force_path_style", "force_path_style"] {
if let Some(Some(force_path_style)) = self.unrecognized.remove(key).map(|v| v.as_bool())
{
let virtual_hosted_style_request = !force_path_style;
self.migrate_aws_value(
"Configuration option",
&format!("pmtiles.{key}"),
"virtual_hosted_style_request",
virtual_hosted_style_request.to_string(),
);
}
}
if let Ok(force_path_style) =
env::var("AWS_S3_FORCE_PATH_STYLE").map(|v| v == "1" || v.to_lowercase() == "true")
{
let virtual_hosted_style_request = !force_path_style;
self.migrate_aws_value(
"Environment variable",
"AWS_S3_FORCE_PATH_STYLE",
"virtual_hosted_style_request",
virtual_hosted_style_request.to_string(),
);
}
for key in ["aws_skip_credentials", "aws_no_credentials"] {
if let Some(Some(no_credentials)) = self.unrecognized.remove(key).map(|v| v.as_bool()) {
self.migrate_aws_value(
"Configuration option",
&format!("pmtiles.{key}"),
"skip_signature",
no_credentials.to_string(),
);
}
}
for env in ["AWS_SKIP_CREDENTIALS", "AWS_NO_CREDENTIALS"] {
if let Ok(skip_credentials) =
env::var(env).map(|v| v == "1" || v.to_lowercase() == "true")
{
self.migrate_aws_value(
"Environment variable",
env,
"skip_signature",
skip_credentials.to_string(),
);
}
}
for env_key in [
"AWS_ACCESS_KEY_ID",
"AWS_SECRET_ACCESS_KEY",
"AWS_SESSION_TOKEN",
"AWS_REGION",
] {
if let Ok(var) = env::var(env_key) {
let new_key_with_aws_prefix = env_key.to_lowercase();
let new_key_without_aws_prefix = new_key_with_aws_prefix
.strip_prefix("aws_")
.expect("all our keys start with aws_");
self.migrate_aws_value(
"Environment variable",
env_key,
new_key_without_aws_prefix,
var,
);
}
}
if env::var("AWS_PROFILE").is_ok() {
warn!(
"Environment variable AWS_PROFILE not supported anymore. Supporting this is in scope, but would need more work. See https://github.com/pola-rs/polars/issues/18757#issuecomment-2379398284"
);
}
}
fn migrate_aws_value(&mut self, r#type: &'static str, key: &str, new_key: &str, value: String) {
let new_key_with_aws_prefix = format!("aws_{new_key}");
if self.options.contains_key(new_key) {
warn!(
"{type} {key} is ignored in favor of the new configuration value pmtiles.{new_key}."
);
} else if self.options.contains_key(&new_key_with_aws_prefix) {
warn!(
"{type} {key} is ignored in favor of the new configuration value pmtiles.{new_key_with_aws_prefix}."
);
} else {
warn!(
"{type} {key} is deprecated. Please use pmtiles.{new_key} in the configuration file instead."
);
self.options.insert(new_key.to_string(), value);
}
}
}
impl TileSourceConfiguration for PmtConfig {
fn parse_urls() -> bool {
true
}
async fn new_sources(&self, id: String, path: PathBuf) -> MartinResult<BoxedSource> {
let path = path
.canonicalize()
.map_err(|e| ConfigFileError::IoError(e, path))?;
let path = std::path::absolute(&path).map_err(|e| ConfigFileError::IoError(e, path))?;
let url = Url::from_file_path(&path)
.or(Err(ConfigFileError::PathNotConvertibleToUrl(path.clone())))?;
trace!(
"Pmtiles source {id} ({}) will be loaded as {url}",
path.display()
);
self.new_sources_url(id, url).await
}
async fn new_sources_url(&self, id: String, url: Url) -> MartinResult<BoxedSource> {
use std::sync::LazyLock;
use std::sync::atomic::{AtomicUsize, Ordering};
static NEXT_CACHE_ID: LazyLock<AtomicUsize> = LazyLock::new(|| AtomicUsize::new(0));
let cache_id = NEXT_CACHE_ID.fetch_add(1, Ordering::SeqCst);
let (store, path) = object_store::parse_url_opts(&url, &self.options)
.map_err(|e| ConfigFileError::ObjectStoreUrlParsing(e, id.clone()))?;
let cache = PmtCacheInstance::new(cache_id, self.pmtiles_directory_cache.clone());
let source = PmtilesSource::new(cache, id, store, path).await?;
Ok(Box::new(source))
}
}