use std::collections::HashMap;
use std::path::{Path, PathBuf};
use std::sync::{Arc, Mutex, RwLock};
use serde::Deserialize;
use crate::error::{Error, Result};
use crate::library::{DEFAULT_TIMEZONE, Library, minor_of, minor_order};
pub const REGISTRY_ENV: &str = "CHTYPES_REGISTRY";
pub const AUTOFETCH_ENV: &str = "CHTYPES_AUTOFETCH";
pub const SYSTEM_ARTIFACT_ROOTS: [&str; 2] = [
"/usr/local/share/chtypes/artifacts",
"/opt/chtypes/artifacts",
];
#[derive(Debug, Clone, Deserialize)]
pub struct Manifest {
pub library: String,
#[serde(default)]
pub library_bytes: u64,
#[serde(default)]
pub library_sha256: String,
#[serde(default)]
pub clickhouse_version: String,
#[serde(default)]
pub clickhouse_minor: String,
#[serde(default)]
pub clickhouse_commit: String,
#[serde(default)]
pub os: String,
#[serde(default)]
pub arch: String,
#[serde(default)]
pub unsafe_families: String,
}
pub struct Registry {
dir: PathBuf,
search: Vec<PathBuf>,
lazy: bool,
#[cfg_attr(not(feature = "fetch"), allow(dead_code))]
explicit: Option<PathBuf>,
timezone: String,
autofetch: bool,
#[cfg(feature = "fetch")]
fetch: crate::fetch::EnsureOptions,
loaded: RwLock<Loaded>,
opening: Mutex<()>,
}
#[derive(Default)]
struct Loaded {
by_id: HashMap<String, Arc<Library>>,
libraries: Vec<Arc<Library>>,
}
impl Loaded {
fn insert(&mut self, library: Arc<Library>) {
self.by_id
.insert(library.version().to_string(), Arc::clone(&library));
self.by_id
.insert(library.minor().to_string(), Arc::clone(&library));
self.libraries.push(library);
self.libraries.sort_by_key(|l| minor_order(l.minor()));
}
fn get(&self, version: &str) -> Option<Arc<Library>> {
if let Some(l) = self.by_id.get(version) {
return Some(Arc::clone(l));
}
self.by_id.get(&minor_of(version)).map(Arc::clone)
}
}
#[derive(Debug, Clone, Default)]
pub struct RegistryOptions {
pub dir: Option<PathBuf>,
pub timezone: Option<String>,
pub autofetch: Option<bool>,
#[cfg(feature = "fetch")]
pub fetch: crate::fetch::EnsureOptions,
}
impl Registry {
pub fn from_search_path() -> Registry {
Registry::from_search_path_with(RegistryOptions::default())
}
pub fn from_search_path_with(opts: RegistryOptions) -> Registry {
let autofetch = opts.autofetch.unwrap_or_else(|| {
std::env::var(AUTOFETCH_ENV)
.map(|v| v == "1")
.unwrap_or(false)
});
Registry {
dir: install_dir(opts.dir.as_deref()),
search: registry_search_path(opts.dir.as_deref()),
lazy: true,
explicit: opts.dir,
timezone: opts
.timezone
.unwrap_or_else(|| DEFAULT_TIMEZONE.to_string()),
autofetch,
#[cfg(feature = "fetch")]
fetch: opts.fetch,
loaded: RwLock::new(Loaded::default()),
opening: Mutex::new(()),
}
}
pub fn new(dir: impl AsRef<Path>) -> Result<Registry> {
Registry::with_timezone(dir, DEFAULT_TIMEZONE)
}
pub fn from_env() -> Result<Registry> {
let dir =
std::env::var_os(REGISTRY_ENV).ok_or(Error::NoRegistryEnv { var: REGISTRY_ENV })?;
Registry::new(PathBuf::from(dir))
}
pub fn from_env_or_default() -> Result<Registry> {
Registry::from_env_or(default_registry_dir())
}
pub fn from_env_or(fallback: impl AsRef<Path>) -> Result<Registry> {
match std::env::var_os(REGISTRY_ENV) {
Some(dir) => Registry::new(PathBuf::from(dir)),
None => Registry::new(fallback),
}
}
pub fn with_timezone(dir: impl AsRef<Path>, timezone: &str) -> Result<Registry> {
let dir = dir.as_ref().to_path_buf();
let mut entries: Vec<PathBuf> = std::fs::read_dir(&dir)
.map_err(|source| Error::Registry {
dir: dir.clone(),
source,
})?
.filter_map(|e| e.ok())
.map(|e| e.path())
.collect();
entries.sort();
let mut loaded = Loaded::default();
for sub in entries {
if !sub.is_dir() {
continue;
}
if let Some(library) = load_artifact_dir(&sub, timezone)? {
loaded.insert(library);
}
}
if loaded.libraries.is_empty() {
return Err(Error::EmptyRegistry { dir });
}
Ok(Registry {
search: vec![dir.clone()],
dir,
lazy: false,
explicit: None,
timezone: timezone.to_string(),
autofetch: false,
#[cfg(feature = "fetch")]
fetch: crate::fetch::EnsureOptions::default(),
loaded: RwLock::new(loaded),
opening: Mutex::new(()),
})
}
pub fn dir(&self) -> &Path {
&self.dir
}
pub fn search_path(&self) -> &[PathBuf] {
&self.search
}
pub fn autofetch(&self) -> bool {
self.autofetch
}
pub fn versions(&self) -> Vec<String> {
let mut out: Vec<String> = self
.loaded()
.libraries
.iter()
.map(|l| l.minor().to_string())
.collect();
if self.lazy {
out.extend(installed_lines(&self.search).into_iter().map(|(m, _)| m));
}
out.sort_by_key(|m| minor_order(m));
out.dedup();
out
}
pub fn libraries(&self) -> Vec<Arc<Library>> {
self.loaded().libraries.clone()
}
fn loaded(&self) -> std::sync::RwLockReadGuard<'_, Loaded> {
self.loaded.read().unwrap_or_else(|p| p.into_inner())
}
pub fn for_version(&self, version: &str) -> Result<Arc<Library>> {
if let Some(l) = self.loaded().get(version) {
return Ok(l);
}
if !self.lazy {
return Err(Error::NoSuchVersion {
requested: version.to_string(),
loaded: self.versions().join(", "),
});
}
let _opening = self.opening.lock().unwrap_or_else(|p| p.into_inner());
if let Some(l) = self.loaded().get(version) {
return Ok(l);
}
let minor = minor_of(version);
let dir = match locate_in(&self.search, &minor) {
Some(dir) => dir,
None => self.autofetch_or_missing(&minor)?,
};
let library = load_artifact_dir(&dir, &self.timezone)?.ok_or_else(|| Error::Registry {
dir: dir.clone(),
source: std::io::Error::new(
std::io::ErrorKind::InvalidData,
"manifest.json is present but names no library",
),
})?;
self.loaded
.write()
.unwrap_or_else(|p| p.into_inner())
.insert(Arc::clone(&library));
Ok(library)
}
fn autofetch_or_missing(&self, minor: &str) -> Result<PathBuf> {
if !self.autofetch {
return Err(self.missing(minor));
}
#[cfg(feature = "fetch")]
{
self.autofetch_line(minor)
}
#[cfg(not(feature = "fetch"))]
{
Err(Error::Fetch {
message: format!(
"autofetch was requested for ClickHouse {minor} but this build of chtypes \
has the `fetch` feature disabled"
),
})
}
}
#[cfg(feature = "fetch")]
fn autofetch_line(&self, minor: &str) -> Result<PathBuf> {
static GUARD: Mutex<std::collections::BTreeSet<String>> =
Mutex::new(std::collections::BTreeSet::new());
let mut attempted = GUARD.lock().unwrap_or_else(|p| p.into_inner());
if let Some(dir) = locate_in(&self.search, minor) {
return Ok(dir);
}
if !attempted.insert(minor.to_string()) {
return Err(self.missing(minor));
}
let opts = crate::fetch::EnsureOptions {
dest: self.explicit.clone(),
..self.fetch.clone()
};
Ok(crate::fetch::ensure(minor, &opts)?.dir)
}
fn missing(&self, minor: &str) -> Error {
Error::ArtifactMissing {
line: minor.to_string(),
platform: host_platform(),
looked_in: self.search.clone(),
}
}
pub fn shutdown(&self) {
for l in &self.loaded().libraries {
l.shutdown();
}
}
}
fn load_artifact_dir(sub: &Path, timezone: &str) -> Result<Option<Arc<Library>>> {
let Ok(text) = std::fs::read_to_string(sub.join("manifest.json")) else {
return Ok(None);
};
let Ok(manifest) = serde_json::from_str::<Manifest>(&text) else {
return Ok(None);
};
if manifest.library.is_empty() {
return Ok(None);
}
let path = sub.join(&manifest.library);
if manifest.library_bytes > 0 {
let actual = std::fs::metadata(&path)
.map_err(|source| Error::Registry {
dir: path.clone(),
source,
})?
.len();
if actual != manifest.library_bytes {
return Err(Error::CorruptArtifact {
path,
expected: manifest.library_bytes,
actual,
});
}
}
let library = Arc::new(Library::load(&path, timezone)?);
if !manifest.clickhouse_version.is_empty() && manifest.clickhouse_version != library.version() {
return Err(Error::VersionMismatch {
path,
reported: library.version().to_string(),
manifest: manifest.clickhouse_version,
});
}
Ok(Some(library))
}
impl std::fmt::Debug for Registry {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("Registry")
.field("dir", &self.dir)
.field("versions", &self.versions())
.finish()
}
}
pub fn host_platform() -> String {
let arch = match std::env::consts::ARCH {
"x86_64" => "amd64",
"aarch64" => "arm64",
other => other,
};
let os = match std::env::consts::OS {
"macos" => "darwin",
other => other,
};
format!("{os}-{arch}")
}
pub fn cache_dir_for(platform: &str) -> PathBuf {
let base = std::env::var_os("XDG_CACHE_HOME")
.map(PathBuf::from)
.filter(|p| !p.as_os_str().is_empty())
.or_else(|| std::env::var_os("HOME").map(|h| PathBuf::from(h).join(".cache")))
.unwrap_or_else(|| PathBuf::from(".cache"));
base.join("chtypes").join("artifacts").join(platform)
}
pub fn default_registry_dir() -> PathBuf {
cache_dir_for(&host_platform())
}
pub fn registry_search_path(explicit: Option<&Path>) -> Vec<PathBuf> {
search_path_for(&host_platform(), explicit)
}
pub fn search_path_for(platform: &str, explicit: Option<&Path>) -> Vec<PathBuf> {
let mut out = Vec::with_capacity(5);
if let Some(d) = explicit {
out.push(d.to_path_buf());
}
if platform == host_platform() {
if let Some(d) = std::env::var_os(REGISTRY_ENV).filter(|v| !v.is_empty()) {
out.push(PathBuf::from(d));
}
}
out.push(cache_dir_for(platform));
for root in SYSTEM_ARTIFACT_ROOTS {
out.push(Path::new(root).join(platform));
}
out
}
pub fn install_dir(explicit: Option<&Path>) -> PathBuf {
install_dir_for(&host_platform(), explicit)
}
pub fn install_dir_for(platform: &str, explicit: Option<&Path>) -> PathBuf {
search_path_for(platform, explicit)
.into_iter()
.next()
.unwrap_or_else(|| cache_dir_for(platform))
}
pub fn locate(line: &str, explicit: Option<&Path>) -> Option<PathBuf> {
locate_in(®istry_search_path(explicit), line)
}
pub fn locate_in(dirs: &[PathBuf], line: &str) -> Option<PathBuf> {
let minor = minor_of(line);
dirs.iter()
.map(|d| d.join(&minor))
.find(|sub| sub.join("manifest.json").is_file())
}
pub fn installed_lines(dirs: &[PathBuf]) -> Vec<(String, PathBuf)> {
let mut seen: Vec<(String, PathBuf)> = Vec::new();
for dir in dirs {
let Ok(entries) = std::fs::read_dir(dir) else {
continue;
};
for entry in entries.filter_map(|e| e.ok()) {
let sub = entry.path();
if !sub.join("manifest.json").is_file() {
continue;
}
let Some(minor) = sub.file_name().and_then(|n| n.to_str()) else {
continue;
};
if minor.starts_with('.') || seen.iter().any(|(m, _)| m == minor) {
continue;
}
seen.push((minor.to_string(), sub));
}
}
seen.sort_by_key(|(m, _)| minor_order(m));
seen
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn a_manifest_parses_and_tolerates_new_fields() {
let m: Manifest = serde_json::from_str(
r#"{"arch":"arm64","clickhouse_commit":"cfec8e0","clickhouse_minor":"25.8",
"clickhouse_version":"25.8.28.1-lts","library":"libchtypes.dylib",
"library_bytes":232226512,"library_sha256":"275c39","os":"darwin",
"unsafe_families":"","some_future_field":true}"#,
)
.unwrap();
assert_eq!(m.library, "libchtypes.dylib");
assert_eq!(m.library_bytes, 232226512);
assert_eq!(m.clickhouse_version, "25.8.28.1-lts");
}
#[test]
fn the_historical_linux_library_name_is_honored() {
let m: Manifest =
serde_json::from_str(r#"{"library":"libchtypes_s1.so","os":"linux"}"#).unwrap();
assert_eq!(m.library, "libchtypes_s1.so");
}
#[test]
fn an_empty_registry_is_an_error_not_an_empty_result() {
let dir = std::env::temp_dir().join(format!("chtypes-rs-empty-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let err = Registry::new(&dir).unwrap_err();
assert!(matches!(err, Error::EmptyRegistry { .. }), "got {err:?}");
std::fs::remove_dir_all(&dir).ok();
}
}