use std::collections::BTreeSet;
use std::path::{Path, PathBuf};
use futures_util::TryStreamExt;
use object_store::ObjectStoreExt;
use object_store::path::Path as ObjectPath;
use crate::adapters::files::constants::{LOAD_ID_LENGTH, MOVED_ASIDE_SUFFIX, STAGED_FOLDER_MARK};
use crate::file_store::models::OpenFolder;
use crate::store_paths::models::{Location, StoreKind};
pub(crate) fn local_folder(location: &Location) -> Option<(PathBuf, &str, &str)> {
let StoreKind::Filesystem { root } = &location.store else {
return None;
};
let trimmed = location.prefix.trim_end_matches('/');
let (parent, name) = match trimmed.rsplit_once('/') {
Some((parent, name)) => (&location.prefix[..=parent.len()], name),
None => ("", trimmed),
};
(!name.is_empty()).then(|| (Path::new(root).join(trimmed), parent, name))
}
pub(crate) fn staged_stem(name: &str) -> String {
format!(".{name}{STAGED_FOLDER_MARK}")
}
pub(crate) fn swap_dirs(staged: &Path, live: &Path) -> std::io::Result<()> {
match live.try_exists()? {
false => std::fs::rename(staged, live),
true => exchange(staged, live).or_else(|error| match error.kind() == std::io::ErrorKind::Unsupported {
true => {
tracing::warn!(%error, folder = %live.display(), "this filesystem cannot swap folders in one rename; moving the old folder aside first");
move_aside(staged, live)
}
false => Err(error),
}),
}
}
fn move_aside(staged: &Path, live: &Path) -> std::io::Result<()> {
let mut aside = staged.as_os_str().to_owned();
aside.push(MOVED_ASIDE_SUFFIX);
let aside = PathBuf::from(aside);
std::fs::rename(live, &aside)?;
if let Err(error) = std::fs::rename(staged, live) {
std::fs::rename(&aside, live)?;
return Err(error);
}
std::fs::rename(&aside, staged)
}
pub(crate) fn staged_entry(entry: &str, stem: &str) -> Option<bool> {
let rest = entry.strip_prefix(stem)?;
let (load_id, aside) = match rest.strip_suffix(MOVED_ASIDE_SUFFIX) {
Some(load_id) => (load_id, true),
None => (rest, false),
};
let is_load_id = load_id.len() == LOAD_ID_LENGTH
&& load_id
.chars()
.all(|character| character.is_ascii_hexdigit() || character == '-');
is_load_id.then_some(aside)
}
#[cfg(any(target_os = "linux", target_os = "macos"))]
fn exchange(staged: &Path, live: &Path) -> std::io::Result<()> {
use rustix::fs::{CWD, RenameFlags, renameat_with};
use rustix::io::Errno;
renameat_with(CWD, staged, CWD, live, RenameFlags::EXCHANGE).map_err(|errno| {
match [Errno::INVAL, Errno::NOSYS, Errno::OPNOTSUPP, Errno::NOTSUP].contains(&errno) {
true => std::io::ErrorKind::Unsupported.into(),
false => errno.into(),
}
})
}
#[cfg(not(any(target_os = "linux", target_os = "macos")))]
fn exchange(_staged: &Path, _live: &Path) -> std::io::Result<()> {
Err(std::io::ErrorKind::Unsupported.into())
}
pub(crate) async fn carry_over(
live: &OpenFolder,
staged: &OpenFolder,
replaced: &BTreeSet<String>,
) -> Result<(), (String, String)> {
let live_prefix = live.location.prefix.as_str();
let exists = local_folder(&live.location).is_some_and(|(path, _, _)| path.exists());
let listed: Vec<object_store::ObjectMeta> = match exists {
true => {
let prefix = ObjectPath::parse(live_prefix.trim_end_matches('/'))
.map_err(|error| (live_prefix.to_owned(), error.to_string()))?;
live.store
.list(Some(&prefix))
.try_collect()
.await
.map_err(|error| (live_prefix.to_owned(), format!("cannot list: {error}")))?
}
false => Vec::new(),
};
for meta in listed {
let Some(key) = kept_key(&meta.location, live_prefix, replaced) else {
continue;
};
let to = ObjectPath::parse(format!("{}{key}", staged.location.prefix))
.map_err(|error| (key.clone(), error.to_string()))?;
copy_file(live, staged, (&meta.location, &to))
.await
.map_err(|error| (key, format!("cannot carry over: {error}")))?;
}
Ok(())
}
fn kept_key(location: &ObjectPath, prefix: &str, replaced: &BTreeSet<String>) -> Option<String> {
let key = location.as_ref().strip_prefix(prefix)?;
(!replaced.contains(key)).then(|| key.to_owned())
}
async fn copy_file(
live: &OpenFolder,
staged: &OpenFolder,
(from, to): (&ObjectPath, &ObjectPath),
) -> Result<(), object_store::Error> {
match staged.store.copy_if_not_exists(from, to).await {
Ok(()) | Err(object_store::Error::AlreadyExists { .. }) => Ok(()),
Err(_) => {
let bytes = live.store.get(from).await?.bytes().await?;
staged.store.put(to, bytes.into()).await.map(|_| ())
}
}
}
pub(crate) fn swap_in(staged: &Path, live: &Path) -> std::io::Result<()> {
std::fs::create_dir_all(staged)?;
swap_dirs(staged, live)?;
let removed = match std::fs::remove_dir_all(staged) {
Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()),
removed => removed,
};
if let Err(error) = removed {
tracing::warn!(%error, folder = %live.display(), "the new table is in place, but the old one beside it could not be deleted; the next load deletes it");
}
Ok(())
}
pub(crate) fn clear_staged(live: &Path, stem: &str) -> std::io::Result<()> {
let Some(parent) = live.parent() else {
return Ok(());
};
let entries: Vec<(PathBuf, bool)> = match std::fs::read_dir(parent) {
Ok(entries) => entries
.filter_map(Result::ok)
.filter_map(|entry| staged_dir(&entry, stem))
.collect(),
Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()),
Err(error) => return Err(error),
};
let mut missing = !live.try_exists()?;
for (entry, aside) in entries {
match aside && missing {
true => {
std::fs::rename(&entry, live)?;
missing = false;
}
false => std::fs::remove_dir_all(&entry)?,
}
}
Ok(())
}
fn staged_dir(entry: &std::fs::DirEntry, stem: &str) -> Option<(PathBuf, bool)> {
staged_entry(&entry.file_name().to_string_lossy(), stem).map(|aside| (entry.path(), aside))
}
pub(crate) async fn remove_staged(live: &Location) {
let Some((path, _, name)) = local_folder(live) else {
return;
};
let stem = staged_stem(name);
let removed = tokio::task::spawn_blocking(move || clear_staged(&path, &stem)).await;
if let Err(error) = removed
.map_err(|error| error.to_string())
.and_then(|removed| removed.map_err(|error| error.to_string()))
{
tracing::warn!(%error, folder = %live, "cannot delete a staging folder an earlier load left beside the table; the next load tries again");
}
}