incpush 0.1.3

Incrementally back up a big local folder to object storage (GCS/S3/Azure), uploading only changed files and preserving previous versions
Documentation
//! Object-storage backend, built on `object_store` so a single implementation
//! serves GCS (`gs://`), S3 (`s3://`), Azure (`az://`), and the local
//! filesystem (`file://`, used by the integration tests). The `gs://`-style URL
//! scheme selects the concrete store; credentials come from the standard
//! environment chains handled by `object_store`.

use anyhow::{Context, Result, bail};
use bytes::Bytes;
use futures::StreamExt;
use object_store::path::Path as ObjPath;
use object_store::{ObjectStore, ObjectStoreExt, PutPayload};
use std::collections::HashMap;
use std::path::Path;
use std::sync::Arc;
use std::time::{SystemTime, UNIX_EPOCH};
use tokio::io::AsyncWriteExt;
use url::Url;

use crate::diff::RESERVED_DIR;

/// A remote object's reconcilable metadata: size and last-modified (unix ns).
pub type RemoteMeta = (u64, i64);

/// Normalize an object key / local path to a canonical form for `--adopt`
/// matching by percent-decoding it. `object_store` may hand back listed keys in
/// raw or percent-encoded form depending on the backend (e.g. `café.txt` vs
/// `caf%C3%A9.txt`); decoding both the listing and the walk path collapses them
/// to the same string so unicode/space names reconcile reliably. Idempotent for
/// already-decoded input.
pub fn decode_key(key: &str) -> String {
    percent_encoding::percent_decode_str(key)
        .decode_utf8_lossy()
        .into_owned()
}

pub struct Backend {
    store: Arc<dyn ObjectStore>,
    /// Base path (the destination directory) within the store.
    prefix: ObjPath,
}

impl Backend {
    /// Build a backend from a destination URL such as `gs://bucket/dir/`.
    ///
    /// Credentials come from the standard per-provider environment variables
    /// (`AWS_*`, `GOOGLE_*`, `AZURE_*`). Note we deliberately do **not** use
    /// `object_store::parse_url`: it constructs stores with `Builder::new()`,
    /// which never reads those env vars — so it can't authenticate against a real
    /// bucket. We build each store via `Builder::from_env()` instead.
    pub fn from_url(dest: &str) -> Result<Self> {
        use object_store::ObjectStoreScheme;
        use object_store::aws::AmazonS3Builder;
        use object_store::azure::MicrosoftAzureBuilder;
        use object_store::gcp::GoogleCloudStorageBuilder;
        use object_store::local::LocalFileSystem;

        let url = Url::parse(dest).with_context(|| format!("invalid destination url: {dest}"))?;
        let (scheme, prefix) = ObjectStoreScheme::parse(&url)
            .with_context(|| format!("unsupported destination url: {dest}"))?;

        let store: Arc<dyn ObjectStore> = match scheme {
            ObjectStoreScheme::AmazonS3 => Arc::new(
                AmazonS3Builder::from_env()
                    .with_url(url.as_str())
                    .build()
                    .with_context(|| format!("configuring S3 store for {dest}"))?,
            ),
            ObjectStoreScheme::GoogleCloudStorage => Arc::new(
                GoogleCloudStorageBuilder::from_env()
                    .with_url(url.as_str())
                    .build()
                    .with_context(|| format!("configuring GCS store for {dest}"))?,
            ),
            ObjectStoreScheme::MicrosoftAzure => Arc::new(
                MicrosoftAzureBuilder::from_env()
                    .with_url(url.as_str())
                    .build()
                    .with_context(|| format!("configuring Azure store for {dest}"))?,
            ),
            ObjectStoreScheme::Local => Arc::new(LocalFileSystem::new()),
            other => bail!("unsupported destination scheme {other:?} for {dest}"),
        };

        Ok(Self { store, prefix })
    }

    /// Resolve a relative key (e.g. `images/a.jpg`) to a full object path.
    fn resolve(&self, rel_key: &str) -> ObjPath {
        let prefix = self.prefix.as_ref();
        if prefix.is_empty() {
            ObjPath::from(rel_key)
        } else {
            ObjPath::from(format!("{prefix}/{rel_key}"))
        }
    }

    /// Turn a full object path into a destination-relative key, or `None` if it
    /// lies outside our prefix.
    fn rel_of(&self, loc: &ObjPath) -> Option<String> {
        let prefix = self.prefix.as_ref();
        let full = loc.as_ref();
        let rel = if prefix.is_empty() {
            full
        } else {
            full.strip_prefix(prefix)?.trim_start_matches('/')
        };
        if rel.is_empty() {
            None
        } else {
            Some(rel.to_string())
        }
    }

    /// List all data objects under the destination (excluding the reserved
    /// prefix) as `rel_path -> (size, last_modified_unix_ns)`. Used by `--adopt`
    /// to reconcile against pre-existing remote data. One-time; holds the listing
    /// in memory.
    ///
    /// Listing a destination with millions of objects paginates for minutes with
    /// no other output, so when `progress` is set we emit a live count to stderr
    /// as pages arrive — otherwise the tool looks hung during a bootstrap adopt.
    pub async fn list_data(&self, progress: bool) -> Result<HashMap<String, RemoteMeta>> {
        use std::io::{IsTerminal, Write};

        let mut out = HashMap::new();
        let mut stream = self.store.list(Some(&self.prefix));
        let mut seen: u64 = 0;
        // Redraw in place on a TTY; fall back to periodic lines when piped/logged.
        let tty = std::io::stderr().is_terminal();
        while let Some(meta) = stream.next().await {
            let meta = meta.context("listing destination")?;
            seen += 1;
            if progress && seen % 10_000 == 0 {
                if tty {
                    eprint!("\rlisting destination… {seen} objects");
                    let _ = std::io::stderr().flush();
                } else {
                    eprintln!("listing destination… {seen} objects");
                }
            }
            let Some(rel) = self.rel_of(&meta.location) else {
                continue;
            };
            if rel.starts_with(RESERVED_DIR) {
                continue;
            }
            let mtime = SystemTime::from(meta.last_modified)
                .duration_since(UNIX_EPOCH)
                .map(|d| d.as_nanos() as i64)
                .unwrap_or(0);
            // Key by the decoded path so it matches the (raw) walk paths.
            out.insert(decode_key(&rel), (meta.size, mtime));
        }
        if progress && tty && seen >= 10_000 {
            eprint!("\r\x1b[K"); // clear the in-place counter; caller prints the total
            let _ = std::io::stderr().flush();
        }
        Ok(out)
    }

    /// Upload an in-memory object (HEAD, deltas, snapshots).
    pub async fn put_bytes(&self, rel_key: &str, bytes: Vec<u8>) -> Result<()> {
        let path = self.resolve(rel_key);
        self.store
            .put(&path, PutPayload::from(Bytes::from(bytes)))
            .await
            .with_context(|| format!("uploading {path}"))?;
        Ok(())
    }

    /// Fetch an object's bytes, or `None` if it does not exist.
    pub async fn get_bytes(&self, rel_key: &str) -> Result<Option<Vec<u8>>> {
        let path = self.resolve(rel_key);
        match self.store.get(&path).await {
            Ok(res) => {
                let bytes = res
                    .bytes()
                    .await
                    .with_context(|| format!("reading {path}"))?;
                Ok(Some(bytes.to_vec()))
            }
            Err(object_store::Error::NotFound { .. }) => Ok(None),
            Err(e) => Err(e).with_context(|| format!("fetching {path}")),
        }
    }

    /// Stream a local file to `rel_key`, using multipart under the hood so large
    /// files never need to be fully buffered in memory.
    pub async fn upload_file(&self, rel_key: &str, abs_path: &Path) -> Result<()> {
        let path = self.resolve(rel_key);
        let mut src = tokio::fs::File::open(abs_path)
            .await
            .with_context(|| format!("opening {} for upload", abs_path.display()))?;
        let mut writer = object_store::buffered::BufWriter::new(self.store.clone(), path.clone());
        tokio::io::copy(&mut src, &mut writer)
            .await
            .with_context(|| format!("streaming {} -> {path}", abs_path.display()))?;
        writer
            .shutdown()
            .await
            .with_context(|| format!("finalizing upload of {path}"))?;
        Ok(())
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    /// A `file://` URL over a temp dir gives us a real ObjectStore to test against.
    fn local_backend(dir: &Path) -> Backend {
        let url = Url::from_directory_path(dir).unwrap();
        Backend::from_url(url.as_str()).unwrap()
    }

    #[tokio::test]
    async fn put_and_get_bytes_roundtrip() {
        let dir = tempfile::tempdir().unwrap();
        let be = local_backend(dir.path());

        assert!(be.get_bytes("missing.txt").await.unwrap().is_none());

        be.put_bytes("a/b/c.json", b"hello".to_vec()).await.unwrap();
        let got = be.get_bytes("a/b/c.json").await.unwrap();
        assert_eq!(got.as_deref(), Some(&b"hello"[..]));

        // Confirms it landed under the destination prefix on disk.
        assert!(dir.path().join("a/b/c.json").exists());
    }

    #[tokio::test]
    async fn list_data_keys_match_local_paths() {
        let dir = tempfile::tempdir().unwrap();
        let be = local_backend(dir.path());
        // Simulate a prior tool's raw-UTF-8 keys plus ascii/space names.
        std::fs::write(dir.path().join("café.txt"), b"x").unwrap(); // raw key
        std::fs::write(dir.path().join("with space.txt"), b"x").unwrap();
        std::fs::create_dir_all(dir.path().join("sub")).unwrap();
        std::fs::write(dir.path().join("sub/a.txt"), b"x").unwrap();
        // An already-percent-encoded key (as a cloud backend's listing may return)
        // must decode back to the real path.
        std::fs::create_dir_all(dir.path().join("enc")).unwrap();
        std::fs::write(dir.path().join("enc/caf%C3%A9.txt"), b"x").unwrap();
        // Reserved state must be excluded.
        std::fs::create_dir_all(dir.path().join(".incpush")).unwrap();
        std::fs::write(dir.path().join(".incpush/HEAD.json"), b"{}").unwrap();

        let map = be.list_data(false).await.unwrap();
        let mut keys: Vec<&String> = map.keys().collect();
        keys.sort();
        // Keys must be the real local paths a walk produces — decoded, never
        // percent-encoded — so --adopt can reconcile against them.
        assert_eq!(
            keys,
            vec!["café.txt", "enc/café.txt", "sub/a.txt", "with space.txt"],
            "list_data keys must be decoded to match walk paths"
        );
    }

    #[tokio::test]
    async fn upload_file_streams_contents() {
        let dir = tempfile::tempdir().unwrap();
        let be = local_backend(dir.path());

        let src = dir.path().join("source.bin");
        let payload = vec![7u8; 5 * 1024 * 1024]; // 5 MiB to exercise multipart buffering
        std::fs::write(&src, &payload).unwrap();

        be.upload_file("uploads/source.bin", &src).await.unwrap();
        let got = be.get_bytes("uploads/source.bin").await.unwrap().unwrap();
        assert_eq!(got.len(), payload.len());
        assert_eq!(got, payload);
    }
}