use std::collections::BTreeMap;
use std::collections::BTreeSet;
use std::path::Path;
use std::path::PathBuf;
use serde::Deserialize;
use serde::Serialize;
use crate::Res;
use crate::checksum::refresh_hash;
use crate::io::storage::Storage;
use crate::lineage::Change;
use crate::lineage::InstalledPackageStatus;
use crate::manifest::Manifest;
use crate::object_hash::ObjectHash;
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub enum PullOutcome {
UpToDate,
CleanUpdate,
KeepsLocalChanges {
added: Vec<PathBuf>,
modified: Vec<PathBuf>,
removed: Vec<PathBuf>,
},
Blocked { conflicts: Vec<PathBuf> },
}
#[allow(clippy::large_enum_variant)]
pub(crate) enum RemoteChange {
Modified(ObjectHash),
Removed,
Added(ObjectHash),
}
pub(crate) fn remote_delta(base: &Manifest, latest: &Manifest) -> BTreeMap<PathBuf, RemoteChange> {
let mut delta = BTreeMap::new();
for base_row in &base.rows {
match latest.get_record(&base_row.logical_key) {
Some(latest_row) if latest_row.hash == base_row.hash => {}
Some(latest_row) => {
delta.insert(
base_row.logical_key.clone(),
RemoteChange::Modified(latest_row.hash.clone()),
);
}
None => {
delta.insert(base_row.logical_key.clone(), RemoteChange::Removed);
}
}
}
for latest_row in &latest.rows {
if base.get_record(&latest_row.logical_key).is_none() {
delta.insert(
latest_row.logical_key.clone(),
RemoteChange::Added(latest_row.hash.clone()),
);
}
}
delta
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct PullPreview {
pub outcome: PullOutcome,
pub added: Vec<PathBuf>,
pub changed: Vec<PathBuf>,
pub removed: Vec<PathBuf>,
pub latest_hash: Option<String>,
}
#[derive(Debug, Default, PartialEq, Eq)]
pub(crate) struct IncomingPaths {
pub added: Vec<PathBuf>,
pub changed: Vec<PathBuf>,
pub removed: Vec<PathBuf>,
}
impl IncomingPaths {
#[must_use]
pub(crate) fn from_delta(delta: &BTreeMap<PathBuf, RemoteChange>) -> Self {
let mut incoming = Self::default();
for (path, change) in delta {
let list = match change {
RemoteChange::Added(_) => &mut incoming.added,
RemoteChange::Modified(_) => &mut incoming.changed,
RemoteChange::Removed => &mut incoming.removed,
};
list.push(path.clone());
}
incoming
}
}
fn same_resulting_content(local: &Change, remote: &RemoteChange, identical: bool) -> bool {
match (local, remote) {
(Change::Modified(row), RemoteChange::Modified(hash))
| (Change::Added(row), RemoteChange::Added(hash)) => identical || &row.hash == hash,
(Change::Removed(_), RemoteChange::Removed) => true,
_ => false,
}
}
#[derive(Debug)]
pub struct Reconciled(BTreeSet<PathBuf>);
impl Reconciled {
#[must_use]
pub fn contains(&self, path: &Path) -> bool {
self.0.contains(path)
}
#[must_use]
pub fn is_empty(&self) -> bool {
self.0.is_empty()
}
#[cfg(test)]
pub(crate) fn unreconciled() -> Self {
Self(BTreeSet::new())
}
}
pub async fn identical_to_latest(
storage: &(impl Storage + Sync),
working_dir: &Path,
status: &InstalledPackageStatus,
base: &Manifest,
latest: &Manifest,
) -> Res<Reconciled> {
let mut identical = BTreeSet::new();
for (path, change) in &status.changes {
let local_hash = match change {
Change::Modified(row) | Change::Added(row) => &row.hash,
Change::Removed(_) => continue,
};
let Some(latest_row) = latest.get_record(path) else {
continue;
};
if base
.get_record(path)
.is_some_and(|base_row| base_row.hash == latest_row.hash)
{
continue;
}
if local_hash.algorithm() == latest_row.hash.algorithm() {
continue;
}
match refresh_hash(storage, &working_dir.join(path), latest_row.clone()).await {
Ok(None) => {
identical.insert(path.clone());
}
Ok(Some(_)) => {}
Err(err) if err.is_not_found() => {}
Err(err) => return Err(err),
}
}
Ok(Reconciled(identical))
}
#[must_use]
pub fn classify_pull(
status: &InstalledPackageStatus,
base: &Manifest,
latest: &Manifest,
identical: &Reconciled,
) -> PullOutcome {
classify_pull_with_delta(status, base, latest, identical, &pull_delta(base, latest))
}
pub(crate) fn pull_delta(base: &Manifest, latest: &Manifest) -> BTreeMap<PathBuf, RemoteChange> {
if base == latest {
BTreeMap::new()
} else {
remote_delta(base, latest)
}
}
#[must_use]
pub(crate) fn classify_pull_with_delta(
status: &InstalledPackageStatus,
base: &Manifest,
latest: &Manifest,
identical: &Reconciled,
delta: &BTreeMap<PathBuf, RemoteChange>,
) -> PullOutcome {
if base == latest {
return PullOutcome::UpToDate;
}
if status.changes.is_empty() {
return PullOutcome::CleanUpdate;
}
let mut conflicts = Vec::new();
let mut added = Vec::new();
let mut modified = Vec::new();
let mut removed = Vec::new();
for (path, change) in &status.changes {
if let Some(remote_change) = delta.get(path) {
if !same_resulting_content(change, remote_change, identical.contains(path)) {
conflicts.push(path.clone());
}
continue;
}
match change {
Change::Added(_) => added.push(path.clone()),
Change::Modified(_) => modified.push(path.clone()),
Change::Removed(_) => removed.push(path.clone()),
}
}
if conflicts.is_empty() {
PullOutcome::KeepsLocalChanges {
added,
modified,
removed,
}
} else {
PullOutcome::Blocked { conflicts }
}
}
#[cfg(test)]
mod tests {
use super::*;
use test_log::test;
use aws_sdk_s3::primitives::ByteStream;
use multihash::Multihash;
use crate::checksum::calculate_hash;
use crate::io::remote::HostChecksums;
use crate::io::remote::HostConfig;
use crate::io::storage::mocks::MockStorage;
use crate::Res;
use crate::lineage::Change;
use crate::lineage::ChangeSet;
use crate::lineage::InstalledPackageStatus;
use crate::lineage::UpstreamState;
use crate::manifest::Manifest;
use crate::manifest::ManifestRow;
fn row(key: &str, hash_seed: &[u8]) -> ManifestRow {
ManifestRow {
logical_key: PathBuf::from(key),
physical_key: format!("s3://b/{key}"),
hash: Multihash::<256>::wrap(0x12, hash_seed)
.unwrap()
.try_into()
.unwrap(),
size: hash_seed.len() as u64,
meta: None,
}
}
fn manifest_of(rows: Vec<ManifestRow>) -> Manifest {
Manifest {
rows,
..Manifest::default()
}
}
#[test]
fn incoming_paths_sorts_each_path_into_one_kind() {
let base = manifest_of(vec![
row("kept.csv", b"1"),
row("dropped.csv", b"2"),
row("edited.csv", b"before"),
]);
let latest = manifest_of(vec![
row("kept.csv", b"1"),
row("edited.csv", b"after"),
row("added.csv", b"3"),
row("also-added.csv", b"4"),
]);
assert_eq!(
IncomingPaths::from_delta(&remote_delta(&base, &latest)),
IncomingPaths {
added: vec![PathBuf::from("added.csv"), PathBuf::from("also-added.csv")],
changed: vec![PathBuf::from("edited.csv")],
removed: vec![PathBuf::from("dropped.csv")],
}
);
}
#[test]
fn a_changed_path_is_not_an_addition() {
let base = manifest_of(vec![row("same-name.csv", b"before")]);
let latest = manifest_of(vec![row("same-name.csv", b"after")]);
let incoming = IncomingPaths::from_delta(&remote_delta(&base, &latest));
assert_eq!(incoming.added, [] as [PathBuf; 0]);
assert_eq!(incoming.changed, vec![PathBuf::from("same-name.csv")]);
assert_eq!(incoming.removed, [] as [PathBuf; 0]);
}
#[test]
fn incoming_paths_are_sorted_by_path() {
let base = manifest_of(vec![
row("z-gone", b"1"),
row("a-gone", b"2"),
row("y-edit", b"3"),
row("b-edit", b"4"),
]);
let latest = manifest_of(vec![
row("y-edit", b"5"),
row("b-edit", b"6"),
row("x-new", b"7"),
row("c-new", b"8"),
]);
let incoming = IncomingPaths::from_delta(&remote_delta(&base, &latest));
assert_eq!(
incoming.added,
vec![PathBuf::from("c-new"), PathBuf::from("x-new")]
);
assert_eq!(
incoming.changed,
vec![PathBuf::from("b-edit"), PathBuf::from("y-edit")]
);
assert_eq!(
incoming.removed,
vec![PathBuf::from("a-gone"), PathBuf::from("z-gone")]
);
}
#[test]
fn a_metadata_only_revision_brings_no_paths() {
let base = manifest_of(vec![row("a", b"1")]);
let mut latest = manifest_of(vec![row("a", b"1")]);
latest.header.message = Some("newer revision message".to_string());
assert_eq!(
IncomingPaths::from_delta(&remote_delta(&base, &latest)),
IncomingPaths::default()
);
}
fn behind(changes: ChangeSet) -> InstalledPackageStatus {
InstalledPackageStatus::new(UpstreamState::Behind, changes)
}
#[test(tokio::test)]
async fn identical_edit_across_algorithms_is_not_a_conflict() -> Res {
let storage = MockStorage::default();
let working_dir = PathBuf::from("/wd");
let path = PathBuf::from("a");
let content = b"exactly the remote's new bytes";
storage
.write_byte_stream(working_dir.join(&path), ByteStream::from_static(content))
.await?;
let latest_row = calculate_hash(
&storage,
&working_dir.join(&path),
&path,
&HostConfig {
checksums: HostChecksums::Crc64,
host: None,
},
)
.await?;
let local_row = calculate_hash(
&storage,
&working_dir.join(&path),
&path,
&HostConfig {
checksums: HostChecksums::Sha256Chunked,
host: None,
},
)
.await?;
assert_ne!(
local_row.hash, latest_row.hash,
"precondition: the two algorithms must disagree on the same bytes"
);
let base = manifest_of(vec![row("a", b"the old content")]);
let latest = manifest_of(vec![latest_row]);
let status = behind(ChangeSet::from([(
path.clone(),
Change::Modified(local_row),
)]));
let identical =
identical_to_latest(&storage, &working_dir, &status, &base, &latest).await?;
let out = classify_pull(&status, &base, &latest, &identical);
assert_eq!(
out,
PullOutcome::KeepsLocalChanges {
added: vec![],
modified: vec![],
removed: vec![],
},
"an edit landing latest's own content is trivially resolved, not a conflict"
);
Ok(())
}
#[test(tokio::test)]
async fn remove_vs_remote_modify_blocks_through_the_pass() -> Res {
let storage = MockStorage::default();
let working_dir = PathBuf::from("/wd");
let path = PathBuf::from("a");
let base = manifest_of(vec![row("a", b"the old content")]);
let latest = manifest_of(vec![row("a", b"what the remote pushed")]);
let status = behind(ChangeSet::from([(
path.clone(),
Change::Removed(row("a", b"the old content")),
)]));
let identical =
identical_to_latest(&storage, &working_dir, &status, &base, &latest).await?;
assert!(
identical.is_empty(),
"a removed path has no content to be identical to"
);
assert_eq!(
classify_pull(&status, &base, &latest, &identical),
PullOutcome::Blocked {
conflicts: vec![path]
}
);
Ok(())
}
#[test(tokio::test)]
async fn both_added_different_content_blocks_through_the_pass() -> Res {
let storage = MockStorage::default();
let working_dir = PathBuf::from("/wd");
let path = PathBuf::from("a");
storage
.write_byte_stream(
working_dir.join(&path),
ByteStream::from_static(b"what this copy added"),
)
.await?;
let local_row = calculate_hash(
&storage,
&working_dir.join(&path),
&path,
&HostConfig {
checksums: HostChecksums::Sha256Chunked,
host: None,
},
)
.await?;
let other = PathBuf::from("other");
storage
.write_byte_stream(
working_dir.join(&other),
ByteStream::from_static(b"what the remote added"),
)
.await?;
let mut latest_row = calculate_hash(
&storage,
&working_dir.join(&other),
&path,
&HostConfig {
checksums: HostChecksums::Crc64,
host: None,
},
)
.await?;
latest_row.logical_key = path.clone();
let base = manifest_of(vec![]);
let latest = manifest_of(vec![latest_row]);
let status = behind(ChangeSet::from([(path.clone(), Change::Added(local_row))]));
let identical =
identical_to_latest(&storage, &working_dir, &status, &base, &latest).await?;
assert!(
identical.is_empty(),
"different bytes are not identical, whatever the algorithms say"
);
assert_eq!(
classify_pull(&status, &base, &latest, &identical),
PullOutcome::Blocked {
conflicts: vec![path]
}
);
Ok(())
}
#[test(tokio::test)]
async fn differing_edit_across_algorithms_still_blocks() -> Res {
let storage = MockStorage::default();
let working_dir = PathBuf::from("/wd");
let path = PathBuf::from("a");
storage
.write_byte_stream(
working_dir.join(&path),
ByteStream::from_static(b"the user's own edit"),
)
.await?;
let local_row = calculate_hash(
&storage,
&working_dir.join(&path),
&path,
&HostConfig {
checksums: HostChecksums::Sha256Chunked,
host: None,
},
)
.await?;
let other = PathBuf::from("other");
storage
.write_byte_stream(
working_dir.join(&other),
ByteStream::from_static(b"what the remote actually pushed"),
)
.await?;
let mut latest_row = calculate_hash(
&storage,
&working_dir.join(&other),
&path,
&HostConfig {
checksums: HostChecksums::Crc64,
host: None,
},
)
.await?;
latest_row.logical_key = path.clone();
let base = manifest_of(vec![row("a", b"the old content")]);
let latest = manifest_of(vec![latest_row]);
let status = behind(ChangeSet::from([(
path.clone(),
Change::Modified(local_row),
)]));
let identical =
identical_to_latest(&storage, &working_dir, &status, &base, &latest).await?;
let out = classify_pull(&status, &base, &latest, &identical);
assert_eq!(
out,
PullOutcome::Blocked {
conflicts: vec![path]
},
"a real both-changed disagreement must still block"
);
Ok(())
}
#[test(tokio::test)]
async fn clean_tree_is_clean_update() -> Res {
let base = manifest_of(vec![row("a", b"1")]);
let latest = manifest_of(vec![row("a", b"2")]); let out = classify_pull(
&behind(ChangeSet::default()),
&base,
&latest,
&Reconciled::unreconciled(),
);
assert_eq!(out, PullOutcome::CleanUpdate);
Ok(())
}
#[test(tokio::test)]
async fn added_file_is_kept() -> Res {
let base = manifest_of(vec![row("a", b"1")]);
let latest = manifest_of(vec![row("a", b"2")]);
let mut changes = ChangeSet::new();
changes.insert(
PathBuf::from("new.txt"),
Change::Added(row("new.txt", b"x")),
);
let out = classify_pull(
&behind(changes),
&base,
&latest,
&Reconciled::unreconciled(),
);
assert_eq!(
out,
PullOutcome::KeepsLocalChanges {
added: vec![PathBuf::from("new.txt")],
modified: vec![],
removed: vec![],
}
);
Ok(())
}
#[test(tokio::test)]
async fn same_path_different_content_blocks() -> Res {
let base = manifest_of(vec![row("a", b"1")]);
let latest = manifest_of(vec![row("a", b"remote")]); let mut changes = ChangeSet::new();
changes.insert(PathBuf::from("a"), Change::Modified(row("a", b"local"))); let out = classify_pull(
&behind(changes),
&base,
&latest,
&Reconciled::unreconciled(),
);
assert_eq!(
out,
PullOutcome::Blocked {
conflicts: vec![PathBuf::from("a")]
}
);
Ok(())
}
#[test(tokio::test)]
async fn identical_edit_is_not_a_conflict() -> Res {
let base = manifest_of(vec![row("a", b"1")]);
let latest = manifest_of(vec![row("a", b"same")]);
let mut changes = ChangeSet::new();
changes.insert(PathBuf::from("a"), Change::Modified(row("a", b"same"))); let out = classify_pull(
&behind(changes),
&base,
&latest,
&Reconciled::unreconciled(),
);
assert_eq!(
out,
PullOutcome::KeepsLocalChanges {
added: vec![],
modified: vec![],
removed: vec![]
}
);
Ok(())
}
#[test(tokio::test)]
async fn local_remove_vs_remote_modify_blocks() -> Res {
let base = manifest_of(vec![row("a", b"1")]);
let latest = manifest_of(vec![row("a", b"2")]); let mut changes = ChangeSet::new();
changes.insert(PathBuf::from("a"), Change::Removed(row("a", b"1"))); let out = classify_pull(
&behind(changes),
&base,
&latest,
&Reconciled::unreconciled(),
);
assert_eq!(
out,
PullOutcome::Blocked {
conflicts: vec![PathBuf::from("a")]
}
);
Ok(())
}
#[test(tokio::test)]
async fn local_modify_vs_remote_remove_blocks() -> Res {
let base = manifest_of(vec![row("a", b"1"), row("b", b"2")]);
let latest = manifest_of(vec![row("b", b"2")]); let mut changes = ChangeSet::new();
changes.insert(PathBuf::from("a"), Change::Modified(row("a", b"local"))); let out = classify_pull(
&behind(changes),
&base,
&latest,
&Reconciled::unreconciled(),
);
assert_eq!(
out,
PullOutcome::Blocked {
conflicts: vec![PathBuf::from("a")]
}
);
Ok(())
}
#[test(tokio::test)]
async fn both_removed_is_not_a_conflict() -> Res {
let base = manifest_of(vec![row("a", b"1"), row("b", b"2")]);
let latest = manifest_of(vec![row("b", b"2")]); let mut changes = ChangeSet::new();
changes.insert(PathBuf::from("a"), Change::Removed(row("a", b"1"))); let out = classify_pull(
&behind(changes),
&base,
&latest,
&Reconciled::unreconciled(),
);
assert_eq!(
out,
PullOutcome::KeepsLocalChanges {
added: vec![],
modified: vec![],
removed: vec![]
}
);
Ok(())
}
#[test(tokio::test)]
async fn both_added_different_content_blocks() -> Res {
let base = manifest_of(vec![row("a", b"1")]);
let latest = manifest_of(vec![row("a", b"1"), row("new.txt", b"remote")]);
let mut changes = ChangeSet::new();
changes.insert(
PathBuf::from("new.txt"),
Change::Added(row("new.txt", b"local")),
);
let out = classify_pull(
&behind(changes),
&base,
&latest,
&Reconciled::unreconciled(),
);
assert_eq!(
out,
PullOutcome::Blocked {
conflicts: vec![PathBuf::from("new.txt")]
}
);
Ok(())
}
#[test(tokio::test)]
async fn both_added_same_content_is_trivially_resolved() -> Res {
let base = manifest_of(vec![row("a", b"1")]);
let latest = manifest_of(vec![row("a", b"1"), row("new.txt", b"same")]);
let mut changes = ChangeSet::new();
changes.insert(
PathBuf::from("new.txt"),
Change::Added(row("new.txt", b"same")),
);
let out = classify_pull(
&behind(changes),
&base,
&latest,
&Reconciled::unreconciled(),
);
assert_eq!(
out,
PullOutcome::KeepsLocalChanges {
added: vec![],
modified: vec![],
removed: vec![],
}
);
Ok(())
}
#[test]
fn the_delta_reports_additions_alongside_modifications_and_removals() {
let base = manifest_of(vec![
row("kept", b"k"),
row("gone", b"g"),
row("edit", b"e1"),
]);
let latest = manifest_of(vec![
row("kept", b"k"),
row("edit", b"e2"),
row("new", b"n"),
]);
let delta = remote_delta(&base, &latest);
assert!(
!delta.contains_key(&PathBuf::from("kept")),
"an unchanged row is not a change"
);
assert!(matches!(
delta.get(&PathBuf::from("gone")),
Some(RemoteChange::Removed)
));
assert!(matches!(
delta.get(&PathBuf::from("edit")),
Some(RemoteChange::Modified(_))
));
assert!(
matches!(delta.get(&PathBuf::from("new")), Some(RemoteChange::Added(h)) if *h == row("new", b"n").hash),
"a latest-only path is an Added carrying its content hash"
);
}
#[test(tokio::test)]
async fn no_remote_change_is_up_to_date() -> Res {
let base = manifest_of(vec![row("a", b"1")]);
let latest = manifest_of(vec![row("a", b"1")]); let out = classify_pull(
&behind(ChangeSet::default()),
&base,
&latest,
&Reconciled::unreconciled(),
);
assert_eq!(out, PullOutcome::UpToDate);
Ok(())
}
#[test(tokio::test)]
async fn metadata_only_change_is_clean_update() -> Res {
let base = manifest_of(vec![row("a", b"1")]);
let mut latest = manifest_of(vec![row("a", b"1")]);
latest.header.message = Some("newer revision message".to_string());
let out = classify_pull(
&behind(ChangeSet::default()),
&base,
&latest,
&Reconciled::unreconciled(),
);
assert_eq!(out, PullOutcome::CleanUpdate);
Ok(())
}
#[test(tokio::test)]
async fn metadata_only_change_keeps_local_changes() -> Res {
let base = manifest_of(vec![row("a", b"1")]);
let mut latest = manifest_of(vec![row("a", b"1")]);
latest.header.message = Some("newer revision message".to_string());
let mut changes = ChangeSet::new();
changes.insert(
PathBuf::from("new.txt"),
Change::Added(row("new.txt", b"x")),
);
let out = classify_pull(
&behind(changes),
&base,
&latest,
&Reconciled::unreconciled(),
);
assert_eq!(
out,
PullOutcome::KeepsLocalChanges {
added: vec![PathBuf::from("new.txt")],
modified: vec![],
removed: vec![],
}
);
Ok(())
}
}