use anyhow::{Context, Result, bail};
use std::cmp::Ordering;
use std::io::{IsTerminal, Write};
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::time::{Instant, SystemTime};
use futures::stream::{FuturesUnordered, StreamExt};
use serde::{Deserialize, Serialize};
use crate::backend::Backend;
use crate::delta::{
DELTA_VERSION, DeltaBuilder, DeltaHeader, DeltaRecord, Head, decode_delta, delta_key,
encode_delta, head_key, snapshot_key,
};
use crate::diff::FileRecord;
use crate::diff::{Decision, DiffMode, RESERVED_DIR, decide, hash_file, walk_source};
use crate::fcm::{Record, path_cmp};
use crate::lsm::Lsm;
use crate::util::human_size;
const TOOL: &str = concat!("incpush/", env!("CARGO_PKG_VERSION"));
const COMPACT_THRESHOLD: usize = 8;
pub struct Config {
pub source: PathBuf,
pub dest: String,
pub mode: DiffMode,
pub excludes: Vec<String>,
pub concurrency: usize,
pub retries: u32,
pub print: bool,
pub quiet: bool,
pub reconcile: bool,
pub no_adopt: bool,
pub no_backupignore: bool,
pub snapshot_every: u32,
pub zstd_level: i32,
}
#[derive(Debug, Default, Clone)]
pub struct Summary {
pub run: i64,
pub total_files: u64,
pub total_size: i64,
pub uploaded_files: u64,
pub uploaded_size: i64,
pub touched: u64,
pub deleted: u64,
pub adopted: u64,
pub failed: u64,
pub snapshot: bool,
pub elapsed: std::time::Duration,
}
#[derive(Debug, Default, Serialize, Deserialize)]
struct LocalState {
last_run: i64,
}
struct Pending {
rel_path: String,
abs_path: PathBuf,
size: i64,
new_record: Record,
}
fn diff_mode_str(m: DiffMode) -> &'static str {
match m {
DiffMode::Mtime => "mtime",
DiffMode::Hash => "hash",
DiffMode::FullHash => "full-hash",
}
}
fn now_rfc3339() -> String {
humantime::format_rfc3339_seconds(SystemTime::now()).to_string()
}
fn read_state(path: &Path) -> i64 {
std::fs::read(path)
.ok()
.and_then(|b| serde_json::from_slice::<LocalState>(&b).ok())
.map(|s| s.last_run)
.unwrap_or(0)
}
fn write_state(path: &Path, last_run: i64) -> Result<()> {
let bytes = serde_json::to_vec_pretty(&LocalState { last_run })?;
std::fs::write(path, bytes).with_context(|| format!("writing {}", path.display()))?;
Ok(())
}
pub async fn run(cfg: Config) -> Result<Summary> {
let started = Instant::now();
let started_ts = now_rfc3339();
let backend = Arc::new(Backend::from_url(&cfg.dest)?);
let state_dir = cfg.source.join(RESERVED_DIR);
let manifest_dir = state_dir.join("manifest");
let state_path = state_dir.join("state.json");
let lsm = Lsm::new(&manifest_dir);
let remote_head = match backend.get_bytes(&head_key()).await? {
Some(bytes) => Some(Head::decode(&bytes)?),
None => None,
};
if !cfg.print {
std::fs::create_dir_all(&manifest_dir)
.with_context(|| format!("creating {}", manifest_dir.display()))?;
}
let last_run = ensure_local_manifest(
&lsm,
&state_path,
&backend,
remote_head.as_ref(),
cfg.print,
cfg.zstd_level,
)
.await?;
let run = last_run + 1;
let is_snapshot = run == 1 || (cfg.snapshot_every > 0 && run % cfg.snapshot_every as i64 == 0);
let bootstrap = remote_head.is_none() && last_run == 0 && !cfg.print;
let adopt_map = if bootstrap && !cfg.no_adopt {
let map = backend.list_data(!cfg.quiet).await?;
if map.is_empty() {
None
} else {
if !cfg.quiet {
eprintln!(
"found {} existing object(s) in the destination; reusing those that match",
map.len()
);
}
Some(map)
}
} else {
None
};
let walk = walk_source(&cfg.source, &cfg.excludes, !cfg.no_backupignore)?;
let mut summary = Summary {
run,
snapshot: is_snapshot,
..Default::default()
};
let mut segment: Vec<Record> = Vec::new(); let mut pending: Vec<Pending> = Vec::new();
let mut deletes: Vec<String> = Vec::new();
classify(
cfg.mode,
cfg.reconcile,
&cfg.source,
lsm.current()?,
walk,
adopt_map.as_ref(),
!cfg.quiet,
&mut summary,
&mut segment,
&mut pending,
&mut deletes,
)?;
if !cfg.quiet && !cfg.print {
let mut line = format!(
"scanned {} files: {} to upload",
summary.total_files,
pending.len()
);
if summary.adopted > 0 {
line.push_str(&format!(", {} adopted", summary.adopted));
}
if summary.deleted > 0 {
line.push_str(&format!(", {} deleted", summary.deleted));
}
eprintln!("{line}");
}
if cfg.print {
for p in &pending {
if !cfg.quiet {
println!("will upload {} ({})", p.rel_path, human_size(p.size));
}
summary.uploaded_files += 1;
summary.uploaded_size += p.size;
}
summary.elapsed = started.elapsed();
return Ok(summary);
}
let results = upload_all(&backend, &pending, cfg.concurrency, cfg.retries, cfg.quiet).await;
let mut all_ok = true;
let mut uploaded_records: Vec<Record> = Vec::new();
for (i, ok) in results.iter().enumerate() {
if *ok {
summary.uploaded_files += 1;
summary.uploaded_size += pending[i].size;
let rec = pending[i].new_record.clone();
uploaded_records.push(rec.clone());
segment.push(rec);
} else {
summary.failed += 1;
all_ok = false;
}
}
if !segment.is_empty() {
lsm.append_segment(&mut segment, cfg.zstd_level)?;
}
if !all_ok {
summary.elapsed = started.elapsed();
bail!(
"{} upload(s) failed; manifest updated so a rerun resumes from here",
summary.failed
);
}
let ended_ts = now_rfc3339();
if is_snapshot {
lsm.compact(cfg.zstd_level)?;
let fcm_bytes = std::fs::read(lsm.base_file())
.with_context(|| "reading compacted base for snapshot".to_string())?;
backend.put_bytes(&snapshot_key(run), fcm_bytes).await?;
let head = Head {
latest_run: run,
base_snapshot: run,
deltas: Vec::new(),
snapshot_every: cfg.snapshot_every,
tool: TOOL.to_string(),
};
backend.put_bytes(&head_key(), head.encode()?).await?;
} else {
let mut builder = DeltaBuilder::new();
for r in &uploaded_records {
builder.upsert(r.path.clone(), r.size, r.mtime, r.crc32c);
}
for d in &deletes {
builder.delete(d.clone());
}
let base_snapshot = remote_head.as_ref().map(|h| h.base_snapshot).unwrap_or(0);
let header = DeltaHeader {
v: DELTA_VERSION,
run,
base_snapshot,
prev_run: last_run,
started: started_ts,
ended: ended_ts,
tool: TOOL.to_string(),
diff: diff_mode_str(cfg.mode).to_string(),
upserts: builder.upserts(),
deletes: builder.deletes(),
};
let bytes = encode_delta(&header, builder.records(), cfg.zstd_level)?;
backend.put_bytes(&delta_key(run), bytes).await?;
let mut deltas = remote_head.map(|h| h.deltas).unwrap_or_default();
deltas.push(run);
let head = Head {
latest_run: run,
base_snapshot,
deltas,
snapshot_every: cfg.snapshot_every,
tool: TOOL.to_string(),
};
backend.put_bytes(&head_key(), head.encode()?).await?;
if lsm.segment_count()? > COMPACT_THRESHOLD {
lsm.compact(cfg.zstd_level)?;
}
}
write_state(&state_path, run)?;
summary.elapsed = started.elapsed();
Ok(summary)
}
#[allow(clippy::too_many_arguments)]
fn progress_tick(tty: bool, msg: &str) {
if tty {
eprint!("\r\x1b[K{msg}");
let _ = std::io::stderr().flush();
} else {
eprintln!("{msg}");
}
}
fn progress_done(tty: bool) {
if tty {
eprint!("\r\x1b[K");
let _ = std::io::stderr().flush();
}
}
#[allow(clippy::too_many_arguments)]
fn classify(
mode: DiffMode,
reconcile: bool,
source: &Path,
mut current: impl Iterator<Item = Result<Record>>,
mut walk: impl Iterator<Item = Result<crate::diff::Entry>>,
adopt: Option<&std::collections::HashMap<String, crate::backend::RemoteMeta>>,
progress: bool,
summary: &mut Summary,
segment: &mut Vec<Record>,
pending: &mut Vec<Pending>,
deletes: &mut Vec<String>,
) -> Result<()> {
enum Step {
Old,
Walk,
Both,
}
let tty = std::io::stderr().is_terminal();
let tick = |n: u64| {
if progress && n % 10_000 == 0 {
progress_tick(tty, &format!("scanning… {n} files"));
}
};
let mut o = current.next().transpose()?;
let mut w = walk.next().transpose()?;
loop {
let step = match (o.as_ref(), w.as_ref()) {
(None, None) => break,
(Some(_), None) => Step::Old,
(None, Some(_)) => Step::Walk,
(Some(orec), Some(e)) => match path_cmp(&orec.path, &e.rel_path) {
Ordering::Less => Step::Old,
Ordering::Greater => Step::Walk,
Ordering::Equal => Step::Both,
},
};
match step {
Step::Old => {
let orec = o.take().unwrap();
if reconcile {
deletes.push(orec.path.clone());
segment.push(Record::tombstone(orec.path));
summary.deleted += 1;
}
o = current.next().transpose()?;
}
Step::Walk => {
let e = w.take().unwrap();
summary.total_files += 1;
summary.total_size += e.size;
tick(summary.total_files);
let adopted = adopt
.and_then(|m| m.get(&crate::backend::decode_key(&e.rel_path)))
.is_some_and(|&(rsize, rmtime)| rsize == e.size as u64 && e.mtime <= rmtime);
if adopted {
summary.adopted += 1;
segment.push(Record {
path: e.rel_path,
size: e.size,
mtime: e.mtime,
crc32c: None,
});
} else {
enqueue_upload(mode, source, &e, pending)?;
}
w = walk.next().transpose()?;
}
Step::Both => {
let orec = o.take().unwrap();
let e = w.take().unwrap();
summary.total_files += 1;
summary.total_size += e.size;
tick(summary.total_files);
let prior = FileRecord {
size: orec.size,
mtime: orec.mtime,
crc32c: orec.crc32c,
};
let decision = decide(mode, Some(&prior), e.size, e.mtime, || {
hash_file(&e.abs_path)
})?;
match decision {
Decision::Skip => {} Decision::Touch { crc32c } => {
summary.touched += 1;
segment.push(Record {
path: e.rel_path,
size: e.size,
mtime: e.mtime,
crc32c,
});
}
Decision::Upload { crc32c } => push_pending(source, e, crc32c, pending),
}
o = current.next().transpose()?;
w = walk.next().transpose()?;
}
}
}
if progress {
progress_done(tty);
}
Ok(())
}
fn enqueue_upload(
mode: DiffMode,
source: &Path,
e: &crate::diff::Entry,
pending: &mut Vec<Pending>,
) -> Result<()> {
let decision = decide(mode, None, e.size, e.mtime, || hash_file(&e.abs_path))?;
if let Decision::Upload { crc32c } = decision {
push_pending(source, e.clone(), crc32c, pending);
}
Ok(())
}
fn push_pending(
source: &Path,
e: crate::diff::Entry,
crc32c: Option<u32>,
pending: &mut Vec<Pending>,
) {
let abs = if e.abs_path.as_os_str().is_empty() {
source.join(&e.rel_path)
} else {
e.abs_path
};
let new_record = Record {
path: e.rel_path.clone(),
size: e.size,
mtime: e.mtime,
crc32c,
};
pending.push(Pending {
rel_path: e.rel_path,
abs_path: abs,
size: e.size,
new_record,
});
}
async fn ensure_local_manifest(
lsm: &Lsm,
state_path: &Path,
backend: &Backend,
remote_head: Option<&Head>,
print: bool,
zstd_level: i32,
) -> Result<i64> {
if !lsm.is_empty()? {
return Ok(read_state(state_path));
}
if print {
return Ok(0);
}
let Some(head) = remote_head else {
return Ok(0); };
let snap_key = snapshot_key(head.base_snapshot);
let snap = backend
.get_bytes(&snap_key)
.await?
.with_context(|| format!("HEAD references missing snapshot {snap_key}"))?;
lsm.install_base(&snap)?;
for run in &head.deltas {
let key = delta_key(*run);
let body = backend
.get_bytes(&key)
.await?
.with_context(|| format!("HEAD references missing delta {key}"))?;
let (_h, records) = decode_delta(&body)?;
let mut seg: Vec<Record> = records
.into_iter()
.map(|rec| match rec {
DeltaRecord::Upsert {
path,
size,
mtime,
crc32c,
} => Record {
path,
size,
mtime,
crc32c,
},
DeltaRecord::Delete { path } => Record::tombstone(path),
})
.collect();
lsm.append_segment(&mut seg, zstd_level)?;
}
lsm.compact(zstd_level)?;
write_state(state_path, head.latest_run)?;
Ok(head.latest_run)
}
async fn upload_all(
backend: &Arc<Backend>,
items: &[Pending],
concurrency: usize,
retries: u32,
quiet: bool,
) -> Vec<bool> {
let mut results = vec![false; items.len()];
let mut inflight = FuturesUnordered::new();
let mut next = 0;
let spawn = |i: usize| {
let backend = backend.clone();
let key = items[i].rel_path.clone();
let abs = items[i].abs_path.clone();
let size = items[i].size;
async move {
let mut attempt = 0;
loop {
match backend.upload_file(&key, &abs).await {
Ok(()) => {
if !quiet {
println!("uploaded {} ({})", key, human_size(size));
}
return (i, true);
}
Err(e) => {
attempt += 1;
if attempt >= retries.max(1) {
eprintln!("failed to upload {key}: {e:#}");
return (i, false);
}
let backoff = std::time::Duration::from_millis(200 * (1 << attempt));
tokio::time::sleep(backoff).await;
}
}
}
}
};
let cap = concurrency.max(1);
while next < items.len() && inflight.len() < cap {
inflight.push(spawn(next));
next += 1;
}
while let Some((i, ok)) = inflight.next().await {
results[i] = ok;
if next < items.len() {
inflight.push(spawn(next));
next += 1;
}
}
results
}