use anyhow::{Context, Result, bail};
use bytes::Bytes;
use futures::StreamExt;
use object_store::path::Path as ObjPath;
use object_store::{ObjectStore, ObjectStoreExt, PutPayload};
use std::collections::HashMap;
use std::path::Path;
use std::sync::Arc;
use std::time::{SystemTime, UNIX_EPOCH};
use tokio::io::AsyncWriteExt;
use url::Url;
use crate::diff::RESERVED_DIR;
pub type RemoteMeta = (u64, i64);
pub fn decode_key(key: &str) -> String {
percent_encoding::percent_decode_str(key)
.decode_utf8_lossy()
.into_owned()
}
pub struct Backend {
store: Arc<dyn ObjectStore>,
prefix: ObjPath,
}
impl Backend {
pub fn from_url(dest: &str) -> Result<Self> {
use object_store::ObjectStoreScheme;
use object_store::aws::AmazonS3Builder;
use object_store::azure::MicrosoftAzureBuilder;
use object_store::gcp::GoogleCloudStorageBuilder;
use object_store::local::LocalFileSystem;
let url = Url::parse(dest).with_context(|| format!("invalid destination url: {dest}"))?;
let (scheme, prefix) = ObjectStoreScheme::parse(&url)
.with_context(|| format!("unsupported destination url: {dest}"))?;
let store: Arc<dyn ObjectStore> = match scheme {
ObjectStoreScheme::AmazonS3 => Arc::new(
AmazonS3Builder::from_env()
.with_url(url.as_str())
.build()
.with_context(|| format!("configuring S3 store for {dest}"))?,
),
ObjectStoreScheme::GoogleCloudStorage => Arc::new(
GoogleCloudStorageBuilder::from_env()
.with_url(url.as_str())
.build()
.with_context(|| format!("configuring GCS store for {dest}"))?,
),
ObjectStoreScheme::MicrosoftAzure => Arc::new(
MicrosoftAzureBuilder::from_env()
.with_url(url.as_str())
.build()
.with_context(|| format!("configuring Azure store for {dest}"))?,
),
ObjectStoreScheme::Local => Arc::new(LocalFileSystem::new()),
other => bail!("unsupported destination scheme {other:?} for {dest}"),
};
Ok(Self { store, prefix })
}
fn resolve(&self, rel_key: &str) -> ObjPath {
let prefix = self.prefix.as_ref();
if prefix.is_empty() {
ObjPath::from(rel_key)
} else {
ObjPath::from(format!("{prefix}/{rel_key}"))
}
}
fn rel_of(&self, loc: &ObjPath) -> Option<String> {
let prefix = self.prefix.as_ref();
let full = loc.as_ref();
let rel = if prefix.is_empty() {
full
} else {
full.strip_prefix(prefix)?.trim_start_matches('/')
};
if rel.is_empty() {
None
} else {
Some(rel.to_string())
}
}
pub async fn list_data(&self, progress: bool) -> Result<HashMap<String, RemoteMeta>> {
use std::io::{IsTerminal, Write};
let mut out = HashMap::new();
let mut stream = self.store.list(Some(&self.prefix));
let mut seen: u64 = 0;
let tty = std::io::stderr().is_terminal();
while let Some(meta) = stream.next().await {
let meta = meta.context("listing destination")?;
seen += 1;
if progress && seen % 10_000 == 0 {
if tty {
eprint!("\rlisting destination… {seen} objects");
let _ = std::io::stderr().flush();
} else {
eprintln!("listing destination… {seen} objects");
}
}
let Some(rel) = self.rel_of(&meta.location) else {
continue;
};
if rel.starts_with(RESERVED_DIR) {
continue;
}
let mtime = SystemTime::from(meta.last_modified)
.duration_since(UNIX_EPOCH)
.map(|d| d.as_nanos() as i64)
.unwrap_or(0);
out.insert(decode_key(&rel), (meta.size, mtime));
}
if progress && tty && seen >= 10_000 {
eprint!("\r\x1b[K"); let _ = std::io::stderr().flush();
}
Ok(out)
}
pub async fn put_bytes(&self, rel_key: &str, bytes: Vec<u8>) -> Result<()> {
let path = self.resolve(rel_key);
self.store
.put(&path, PutPayload::from(Bytes::from(bytes)))
.await
.with_context(|| format!("uploading {path}"))?;
Ok(())
}
pub async fn get_bytes(&self, rel_key: &str) -> Result<Option<Vec<u8>>> {
let path = self.resolve(rel_key);
match self.store.get(&path).await {
Ok(res) => {
let bytes = res
.bytes()
.await
.with_context(|| format!("reading {path}"))?;
Ok(Some(bytes.to_vec()))
}
Err(object_store::Error::NotFound { .. }) => Ok(None),
Err(e) => Err(e).with_context(|| format!("fetching {path}")),
}
}
pub async fn upload_file(&self, rel_key: &str, abs_path: &Path) -> Result<()> {
let path = self.resolve(rel_key);
let mut src = tokio::fs::File::open(abs_path)
.await
.with_context(|| format!("opening {} for upload", abs_path.display()))?;
let mut writer = object_store::buffered::BufWriter::new(self.store.clone(), path.clone());
tokio::io::copy(&mut src, &mut writer)
.await
.with_context(|| format!("streaming {} -> {path}", abs_path.display()))?;
writer
.shutdown()
.await
.with_context(|| format!("finalizing upload of {path}"))?;
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
fn local_backend(dir: &Path) -> Backend {
let url = Url::from_directory_path(dir).unwrap();
Backend::from_url(url.as_str()).unwrap()
}
#[tokio::test]
async fn put_and_get_bytes_roundtrip() {
let dir = tempfile::tempdir().unwrap();
let be = local_backend(dir.path());
assert!(be.get_bytes("missing.txt").await.unwrap().is_none());
be.put_bytes("a/b/c.json", b"hello".to_vec()).await.unwrap();
let got = be.get_bytes("a/b/c.json").await.unwrap();
assert_eq!(got.as_deref(), Some(&b"hello"[..]));
assert!(dir.path().join("a/b/c.json").exists());
}
#[tokio::test]
async fn list_data_keys_match_local_paths() {
let dir = tempfile::tempdir().unwrap();
let be = local_backend(dir.path());
std::fs::write(dir.path().join("café.txt"), b"x").unwrap(); std::fs::write(dir.path().join("with space.txt"), b"x").unwrap();
std::fs::create_dir_all(dir.path().join("sub")).unwrap();
std::fs::write(dir.path().join("sub/a.txt"), b"x").unwrap();
std::fs::create_dir_all(dir.path().join("enc")).unwrap();
std::fs::write(dir.path().join("enc/caf%C3%A9.txt"), b"x").unwrap();
std::fs::create_dir_all(dir.path().join(".incpush")).unwrap();
std::fs::write(dir.path().join(".incpush/HEAD.json"), b"{}").unwrap();
let map = be.list_data(false).await.unwrap();
let mut keys: Vec<&String> = map.keys().collect();
keys.sort();
assert_eq!(
keys,
vec!["café.txt", "enc/café.txt", "sub/a.txt", "with space.txt"],
"list_data keys must be decoded to match walk paths"
);
}
#[tokio::test]
async fn upload_file_streams_contents() {
let dir = tempfile::tempdir().unwrap();
let be = local_backend(dir.path());
let src = dir.path().join("source.bin");
let payload = vec![7u8; 5 * 1024 * 1024]; std::fs::write(&src, &payload).unwrap();
be.upload_file("uploads/source.bin", &src).await.unwrap();
let got = be.get_bytes("uploads/source.bin").await.unwrap().unwrap();
assert_eq!(got.len(), payload.len());
assert_eq!(got, payload);
}
}