incpush 0.1.3

Incrementally back up a big local folder to object storage (GCS/S3/Azure), uploading only changed files and preserving previous versions
Documentation
//! Per-run delta format and remote state pointers.
//!
//! The local manifest is never uploaded whole on the hot path. Instead each run
//! ships a small zstd-compressed NDJSON delta listing only changed/deleted
//! files, plus a tiny `HEAD.json` pointer. A full manifest snapshot is uploaded
//! periodically to bound reconstruction cost. See the crate README for the wire
//! layout under the reserved `.incpush/` prefix.

use anyhow::{Context, Result, bail};
use serde::{Deserialize, Serialize};

/// Current on-disk delta format version.
pub const DELTA_VERSION: u32 = 1;

/// First line of every delta file: run-level metadata and record counts.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct DeltaHeader {
    pub v: u32,
    pub run: i64,
    /// Snapshot run this delta chain is based on.
    pub base_snapshot: i64,
    /// The immediately preceding run (for gap detection); 0 if none.
    pub prev_run: i64,
    pub started: String,
    pub ended: String,
    pub tool: String,
    /// Diff mode used: `"mtime"`, `"hash"`, or `"full-hash"`.
    pub diff: String,
    pub upserts: u64,
    pub deletes: u64,
}

/// A single change record within a delta. Serialized one-per-line as NDJSON.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(tag = "op")]
pub enum DeltaRecord {
    /// Created or changed file.
    #[serde(rename = "U")]
    Upsert {
        path: String,
        size: i64,
        mtime: i64,
        #[serde(default, skip_serializing_if = "Option::is_none")]
        crc32c: Option<u32>,
    },
    /// File removed from source (only emitted under `--reconcile`).
    #[serde(rename = "D")]
    Delete { path: String },
}

impl DeltaRecord {
    pub fn is_delete(&self) -> bool {
        matches!(self, DeltaRecord::Delete { .. })
    }
}

/// Accumulates records for a run and tracks counts for the header.
#[derive(Default)]
pub struct DeltaBuilder {
    records: Vec<DeltaRecord>,
    upserts: u64,
    deletes: u64,
}

impl DeltaBuilder {
    pub fn new() -> Self {
        Self::default()
    }

    pub fn upsert(&mut self, path: String, size: i64, mtime: i64, crc32c: Option<u32>) {
        self.records.push(DeltaRecord::Upsert {
            path,
            size,
            mtime,
            crc32c,
        });
        self.upserts += 1;
    }

    pub fn delete(&mut self, path: String) {
        self.records.push(DeltaRecord::Delete { path });
        self.deletes += 1;
    }

    pub fn upserts(&self) -> u64 {
        self.upserts
    }

    pub fn deletes(&self) -> u64 {
        self.deletes
    }

    pub fn is_empty(&self) -> bool {
        self.records.is_empty()
    }

    pub fn records(&self) -> &[DeltaRecord] {
        &self.records
    }
}

/// Serialize header + records as NDJSON and zstd-compress at `level`.
pub fn encode_delta(header: &DeltaHeader, records: &[DeltaRecord], level: i32) -> Result<Vec<u8>> {
    let mut out = String::new();
    out.push_str(&serde_json::to_string(header)?);
    out.push('\n');
    for r in records {
        out.push_str(&serde_json::to_string(r)?);
        out.push('\n');
    }
    zstd::encode_all(out.as_bytes(), level).context("zstd-compressing delta")
}

/// Decompress and parse a delta file back into its header and records.
pub fn decode_delta(bytes: &[u8]) -> Result<(DeltaHeader, Vec<DeltaRecord>)> {
    let raw = zstd::decode_all(bytes).context("zstd-decompressing delta")?;
    let text = String::from_utf8(raw).context("delta is not valid UTF-8")?;
    let mut lines = text.lines();
    let header_line = lines.next().context("empty delta file")?;
    let header: DeltaHeader = serde_json::from_str(header_line).context("parsing delta header")?;
    if header.v != DELTA_VERSION {
        bail!(
            "unsupported delta version {} (this build understands {})",
            header.v,
            DELTA_VERSION
        );
    }
    let mut records = Vec::with_capacity((header.upserts + header.deletes) as usize);
    for (i, line) in lines.enumerate() {
        if line.is_empty() {
            continue;
        }
        records.push(
            serde_json::from_str(line)
                .with_context(|| format!("parsing delta record on line {}", i + 2))?,
        );
    }
    Ok((header, records))
}

/// Remote pointer to the current backup state. Small; overwritten every run.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct Head {
    pub latest_run: i64,
    pub base_snapshot: i64,
    /// Delta run ids applied on top of `base_snapshot`, in apply order.
    pub deltas: Vec<i64>,
    pub snapshot_every: u32,
    pub tool: String,
}

impl Head {
    pub fn encode(&self) -> Result<Vec<u8>> {
        Ok(serde_json::to_vec_pretty(self)?)
    }

    pub fn decode(bytes: &[u8]) -> Result<Self> {
        serde_json::from_slice(bytes).context("parsing HEAD.json")
    }
}

// --- Object keys, relative to the destination prefix ---------------------------

use crate::diff::RESERVED_DIR;

pub fn head_key() -> String {
    format!("{RESERVED_DIR}/HEAD.json")
}

pub fn delta_key(run: i64) -> String {
    format!("{RESERVED_DIR}/deltas/run-{run}.delta.zst")
}

pub fn snapshot_key(run: i64) -> String {
    format!("{RESERVED_DIR}/snapshots/manifest-{run}.fcm")
}

#[cfg(test)]
mod tests {
    use super::*;

    fn sample_header(up: u64, del: u64) -> DeltaHeader {
        DeltaHeader {
            v: DELTA_VERSION,
            run: 42,
            base_snapshot: 0,
            prev_run: 41,
            started: "2026-07-21T10:00:00Z".into(),
            ended: "2026-07-21T10:03:12Z".into(),
            tool: "incpush/0.1.0".into(),
            diff: "mtime".into(),
            upserts: up,
            deletes: del,
        }
    }

    #[test]
    fn delta_roundtrip() {
        let mut b = DeltaBuilder::new();
        b.upsert("images/a.jpg".into(), 100, 1234, None);
        b.upsert("data/b.bin".into(), 200, 5678, Some(0xabcd));
        b.delete("old/gone.txt".into());
        assert_eq!(b.upserts(), 2);
        assert_eq!(b.deletes(), 1);

        let header = sample_header(b.upserts(), b.deletes());
        let bytes = encode_delta(&header, b.records(), 6).unwrap();
        let (h2, recs) = decode_delta(&bytes).unwrap();

        assert_eq!(h2, header);
        assert_eq!(recs, b.records());
        assert_eq!(recs.iter().filter(|r| r.is_delete()).count(), 1);
    }

    #[test]
    fn upsert_without_crc_omits_field_in_json() {
        let r = DeltaRecord::Upsert {
            path: "x".into(),
            size: 1,
            mtime: 2,
            crc32c: None,
        };
        let s = serde_json::to_string(&r).unwrap();
        assert!(!s.contains("crc32c"), "None crc should be skipped: {s}");
        assert!(s.contains("\"op\":\"U\""));
    }

    #[test]
    fn decode_rejects_wrong_version() {
        let mut header = sample_header(0, 0);
        header.v = 999;
        let bytes = encode_delta(&header, &[], 6).unwrap();
        let err = decode_delta(&bytes).unwrap_err();
        assert!(err.to_string().contains("unsupported delta version"));
    }

    #[test]
    fn head_roundtrip() {
        let head = Head {
            latest_run: 42,
            base_snapshot: 0,
            deltas: vec![1, 2, 3, 42],
            snapshot_every: 100,
            tool: "incpush/0.1.0".into(),
        };
        let bytes = head.encode().unwrap();
        assert_eq!(Head::decode(&bytes).unwrap(), head);
    }

    #[test]
    fn object_keys_are_namespaced() {
        assert_eq!(head_key(), ".incpush/HEAD.json");
        assert_eq!(delta_key(7), ".incpush/deltas/run-7.delta.zst");
        assert_eq!(snapshot_key(100), ".incpush/snapshots/manifest-100.fcm");
    }
}