use std::collections::{BTreeMap, BTreeSet};
use std::io::Write as _;
use std::path::Path;
use std::process::Stdio;
use anyhow::{Context as _, Result, ensure};
use serde_json::json;
use tokio::io::AsyncReadExt as _;
use crate::proc::Quiet as _;
use crate::{git, run::RunState};
const MAX_BYTES: u64 = 64 * 1024 * 1024;
const MAX_FILES: usize = 4096;
const MAX_HEADS: usize = 128;
pub(crate) async fn preserve(
state: &RunState,
home: &Path,
heads: &BTreeSet<String>,
) -> Result<()> {
ensure!(
heads.len() <= MAX_HEADS,
"fold evidence source limit exceeded"
);
let dir = home
.join("runs")
.join(&state.id)
.join("artifacts/fold-evidence");
std::fs::create_dir_all(&dir)?;
let artifacts = dir.parent().context("artifacts directory")?;
sync_directory(artifacts)?;
sync_directory(artifacts.parent().context("run directory")?)?;
let mut bytes = 0;
let mut files = 0;
for entry in std::fs::read_dir(&dir)? {
let entry = entry?;
if entry.file_name().to_string_lossy().ends_with(".pending") {
std::fs::remove_file(entry.path())?;
continue;
}
let metadata = entry.metadata()?;
ensure!(metadata.is_file(), "unexpected fold evidence directory");
bytes += metadata.len();
files += 1;
}
ensure!(
bytes <= MAX_BYTES && files <= MAX_FILES,
"fold evidence storage limit exceeded"
);
let mut manifest: BTreeMap<String, serde_json::Value> =
match std::fs::read(dir.join("manifest.json")) {
Ok(data) => serde_json::from_slice(&data)?,
Err(error) if error.kind() == std::io::ErrorKind::NotFound => BTreeMap::new(),
Err(error) => return Err(error.into()),
};
let unincorporated = unincorporated_heads(state, heads).await?;
for head in &unincorporated {
let key = format!("head-{head}");
let patch = source_patch(&state.repo, &state.base_commit, head).await?;
let index = source_index(&state.repo, &state.base_commit, head).await?;
let mut blobs = BTreeSet::new();
let mut records = index.split(|b| *b == 0);
while let Some(header) = records.next() {
if header.is_empty() {
break;
}
records.next().context("missing raw fold pathname")?;
let header = std::str::from_utf8(header)?;
let fields: Vec<_> = header.split_whitespace().collect();
ensure!(fields.len() == 5, "invalid raw fold change index");
let old = fields[2];
if fields[0] != ":160000" && !old.bytes().all(|b| b == b'0') {
blobs.insert(old.to_owned());
}
}
for blob in &blobs {
let data = capture(&state.repo, &["cat-file", "blob", blob]).await?;
store(&dir, &format!("blob-{blob}"), &data, &mut bytes, &mut files)?;
}
let patch_name = format!("{head}.patch");
let index_name = format!("{head}.index");
store(&dir, &patch_name, &patch, &mut bytes, &mut files)?;
store(&dir, &index_name, &index, &mut bytes, &mut files)?;
manifest.insert(
key,
json!({
"head": head, "base_commit": state.base_commit,
"patch": patch_name, "index": index_name, "base_blobs": blobs,
"patch_bytes": patch.len(), "index_bytes": index.len(),
}),
);
}
if !manifest.contains_key("sources") {
let mut sources = Vec::new();
for candidate in &state.candidates {
let head = if git::branch_exists(&state.repo, &candidate.branch).await? {
Some(git::rev_parse(&state.repo, &candidate.branch).await?)
} else {
None
};
sources.push(json!({
"head": head,
"label": candidate.label, "branch": candidate.branch,
"missing": !candidate.worktree.try_exists()?,
"released": state.released_branches.contains(&candidate.branch),
}));
}
let entries = match std::fs::read_dir(state.worktree_root()) {
Ok(entries) => Some(entries),
Err(error) if error.kind() == std::io::ErrorKind::NotFound => None,
Err(error) => return Err(error.into()),
};
for entry in entries.into_iter().flatten() {
let entry = entry?;
sources.push(json!({
"worktree": entry.file_name().to_string_lossy(),
"head": git::rev_parse(&entry.path(), "HEAD").await?,
}));
}
manifest.insert("sources".into(), json!(sources));
manifest.insert(
"history_policy".into(),
json!("Unincorporated current source tips preserved; obsolete reflog/ORIG_HEAD history discarded."),
);
}
let data = serde_json::to_vec_pretty(&manifest)?;
let old = std::fs::metadata(dir.join("manifest.json")).map_or(0, |m| m.len());
ensure!(
bytes - old + data.len() as u64 <= MAX_BYTES,
"fold evidence byte limit exceeded"
);
ensure!(
files + usize::from(!dir.join("manifest.json").try_exists()?) <= MAX_FILES,
"fold evidence file limit exceeded"
);
atomic_write(&dir.join("manifest.json"), &data)?;
Ok(())
}
async fn unincorporated_heads(
state: &RunState,
heads: &BTreeSet<String>,
) -> Result<BTreeSet<String>> {
if heads.is_empty() {
return Ok(BTreeSet::new());
}
let reference = format!(
"refs/remotes/{}/{}",
state.config.merge.remote, state.base_branch
);
let base = git::rev_parse(&state.repo, &reference)
.await
.context("cannot prove merged sources against the fetched base")?;
let mut squash_winner = None;
let mut checked_squash = false;
let mut remaining = BTreeSet::new();
for head in heads {
if is_ancestor(&state.repo, head, &base).await? {
continue;
}
if !checked_squash {
squash_winner = proven_squash_winner(state, &base).await?;
checked_squash = true;
}
if let Some(winner) = &squash_winner
&& is_ancestor(&state.repo, head, winner).await?
{
continue;
}
remaining.insert(head.clone());
}
Ok(remaining)
}
async fn is_ancestor(repo: &Path, head: &str, base: &str) -> Result<bool> {
Ok(
git::git_raw(repo, &["merge-base", "--is-ancestor", head, base])
.await?
.ok(),
)
}
async fn proven_squash_winner(state: &RunState, base: &str) -> Result<Option<String>> {
let Some(winner) = state.winner() else {
return Ok(None);
};
if !git::branch_exists(&state.repo, &winner.branch).await? {
return Ok(None);
}
let head = git::rev_parse(&state.repo, &winner.branch).await?;
if !is_ancestor(&state.repo, &state.base_commit, &head).await? {
return Ok(None);
}
let patch = source_patch(&state.repo, &state.base_commit, &head).await?;
if patch.is_empty() {
return Ok(None);
}
let range = format!("{}..{base}", state.base_commit);
let commits = capture(&state.repo, &["log", "--no-merges", "--format=%H", &range]).await?;
for commit in std::str::from_utf8(&commits)?.lines() {
let parent = format!("{commit}^");
if source_patch(&state.repo, &parent, commit).await? == patch {
return Ok(Some(head));
}
}
Ok(None)
}
async fn source_patch(repo: &Path, from: &str, to: &str) -> Result<Vec<u8>> {
capture(
repo,
&[
"-c",
"core.quotePath=true",
"diff",
"--binary",
"--full-index",
"--no-renames",
"--no-ext-diff",
"--no-textconv",
"--no-color",
"--ignore-submodules=none",
"--submodule=short",
from,
to,
"--",
],
)
.await
}
async fn source_index(repo: &Path, from: &str, to: &str) -> Result<Vec<u8>> {
capture(
repo,
&[
"diff",
"--raw",
"--no-ext-diff",
"--no-textconv",
"--no-abbrev",
"--no-renames",
"-z",
from,
to,
"--",
],
)
.await
}
async fn capture(repo: &Path, args: &[&str]) -> Result<Vec<u8>> {
let mut child = tokio::process::Command::new("git")
.quiet()
.args(args)
.current_dir(repo)
.stdin(Stdio::null())
.stdout(Stdio::piped())
.stderr(Stdio::null())
.kill_on_drop(true)
.spawn()?;
let mut data = Vec::new();
child
.stdout
.take()
.context("git stdout")?
.take(MAX_BYTES + 1)
.read_to_end(&mut data)
.await?;
ensure!(
data.len() as u64 <= MAX_BYTES,
"fold evidence byte limit exceeded"
);
ensure!(
child.wait().await?.success(),
"git fold evidence capture failed: {args:?}"
);
Ok(data)
}
fn store(dir: &Path, name: &str, data: &[u8], bytes: &mut u64, files: &mut usize) -> Result<()> {
let path = dir.join(name);
if path.try_exists()? {
ensure!(
std::fs::read(&path)? == data,
"fold evidence content mismatch: {name}"
);
return Ok(());
}
ensure!(
*files < MAX_FILES && *bytes + data.len() as u64 <= MAX_BYTES,
"fold evidence storage limit exceeded"
);
atomic_write(&path, data)?;
*bytes += data.len() as u64;
*files += 1;
Ok(())
}
pub(crate) fn atomic_write(path: &Path, data: &[u8]) -> Result<()> {
let tmp = path.with_file_name(format!(
"{}.pending",
path.file_name().context("artifact name")?.to_string_lossy()
));
let mut opened = false;
let result = (|| {
if let Ok(meta) = std::fs::symlink_metadata(&tmp) {
ensure!(
meta.is_file() && !meta.file_type().is_symlink(),
"non-file evidence staging path"
);
}
let mut options = std::fs::OpenOptions::new();
options.write(true).create(true).truncate(true);
#[cfg(unix)]
{
use std::os::unix::fs::OpenOptionsExt as _;
options.custom_flags(libc::O_NOFOLLOW);
}
let mut file = options.open(&tmp)?;
opened = true;
file.write_all(data)?;
file.sync_all()?;
std::fs::rename(&tmp, path)?;
sync_directory(path.parent().context("artifact parent")?)?;
Ok(())
})();
if result.is_err() && opened {
let _ = std::fs::remove_file(&tmp);
}
result
}
pub(crate) fn sync_directory(_path: &Path) -> Result<()> {
#[cfg(unix)]
std::fs::File::open(_path)?.sync_all()?;
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn inspection_creation_retries_collisions_without_adopting_existing_paths() {
let temp = tempfile::tempdir().unwrap();
let parent = temp.path().canonicalize().unwrap();
let directory = parent.join("magi-source-inspection-0000000000000001");
let file = parent.join("magi-source-inspection-0000000000000002");
std::fs::create_dir(&directory).unwrap();
std::fs::write(directory.join("keep"), b"directory evidence").unwrap();
std::fs::write(&file, b"file evidence").unwrap();
let mut next = 0;
let inspection = Inspection::create_with(&parent, || {
next += 1;
next
})
.unwrap();
assert_eq!(next, 3);
assert_eq!(
inspection.0,
parent.join("magi-source-inspection-0000000000000003")
);
assert!(inspection.0.is_dir());
let owned = inspection.0.clone();
drop(inspection);
assert!(!owned.exists());
assert_eq!(
std::fs::read(directory.join("keep")).unwrap(),
b"directory evidence"
);
assert_eq!(std::fs::read(file).unwrap(), b"file evidence");
}
#[test]
fn inspection_creation_propagates_other_filesystem_errors() {
let temp = tempfile::tempdir().unwrap();
let file = temp.path().join("file");
std::fs::write(&file, b"keep").unwrap();
let mut calls = 0;
assert!(
Inspection::create_with(&file, || {
calls += 1;
1
})
.is_err()
);
assert_eq!(calls, 1);
assert_eq!(std::fs::read(file).unwrap(), b"keep");
}
#[tokio::test]
async fn inspection_initialization_preserves_original_blob_line_endings() {
let temp = tempfile::tempdir().unwrap();
let repo = temp.path().canonicalize().unwrap();
init_inspection(&repo).await.unwrap();
git::git(&repo, &["config", "core.autocrlf", "true"])
.await
.unwrap();
git::git(&repo, &["config", "core.eol", "crlf"])
.await
.unwrap();
init_inspection(&repo).await.unwrap();
let object = input(
&repo,
&["hash-object", "-w", "--stdin"],
b"original\n".to_vec(),
)
.await
.unwrap();
let object = String::from_utf8(object).unwrap();
git::git(
&repo,
&[
"update-index",
"--add",
"--cacheinfo",
"100644",
object.trim(),
"line.txt",
],
)
.await
.unwrap();
git::git(&repo, &["checkout-index", "--all"]).await.unwrap();
assert_eq!(std::fs::read(repo.join("line.txt")).unwrap(), b"original\n");
}
#[cfg(unix)]
#[test]
fn evidence_staging_preserves_unknown_symlinks() -> Result<()> {
let temp = tempfile::tempdir()?;
let target = temp.path().join("operator-file");
std::fs::write(&target, "keep")?;
let artifact = temp.path().join("artifact");
let pending = temp.path().join("artifact.pending");
std::os::unix::fs::symlink(&target, &pending)?;
assert!(atomic_write(&artifact, b"new evidence").is_err());
assert!(
std::fs::symlink_metadata(&pending)?
.file_type()
.is_symlink()
);
assert_eq!(std::fs::read(&target)?, b"keep");
Ok(())
}
#[test]
fn evidence_publication_syncs_supported_directories() -> Result<()> {
let dir = tempfile::tempdir()?;
let artifacts = dir.path().join("artifacts");
let evidence = artifacts.join("fold-evidence");
std::fs::create_dir_all(&evidence)?;
sync_directory(&artifacts)?;
sync_directory(dir.path())?;
let path = evidence.join("manifest.json");
atomic_write(&path, b"{}")?;
assert_eq!(std::fs::read(&path)?, b"{}");
assert!(!evidence.join("manifest.json.pending").exists());
Ok(())
}
}
#[derive(serde::Serialize, serde::Deserialize)]
#[serde(deny_unknown_fields)]
struct Snapshot {
version: u32,
#[serde(default, skip_serializing_if = "Option::is_none")]
anchor: Option<String>,
head: String,
base: String,
head_tree: String,
head_commit: Vec<u8>,
base_commit: Vec<u8>,
base_tree: String,
pack_hash: String,
patch_hash: String,
index_hash: String,
}
async fn input(repo: &Path, args: &[&str], data: Vec<u8>) -> Result<Vec<u8>> {
use tokio::io::AsyncWriteExt as _;
let mut child = tokio::process::Command::new("git")
.quiet()
.args(args)
.current_dir(repo)
.stdin(Stdio::piped())
.stdout(Stdio::piped())
.stderr(Stdio::null())
.kill_on_drop(true)
.spawn()?;
let mut stdin = child.stdin.take().context("git stdin")?;
let mut stdout = child
.stdout
.take()
.context("git stdout")?
.take(MAX_BYTES + 1);
let mut output = Vec::new();
let (write, read) = tokio::join!(
async move { stdin.write_all(&data).await },
stdout.read_to_end(&mut output)
);
write?;
read?;
ensure!(
output.len() as u64 <= MAX_BYTES,
"snapshot byte limit exceeded"
);
ensure!(
child.wait().await?.success(),
"snapshot git command failed: {args:?}"
);
Ok(output)
}
async fn digest(repo: &Path, bytes: Vec<u8>) -> Result<String> {
Ok(
String::from_utf8(input(repo, &["hash-object", "--stdin"], bytes).await?)?
.trim()
.to_owned(),
)
}
fn plain(path: &Path) -> Result<()> {
let meta = std::fs::symlink_metadata(path)?;
ensure!(
(meta.is_file() || meta.is_dir())
&& !meta.file_type().is_symlink()
&& path.canonicalize()? == path,
"noncanonical source evidence"
);
Ok(())
}
pub(crate) async fn snapshot(
state: &RunState,
home: &Path,
head: &str,
budget: u64,
) -> Result<std::path::PathBuf> {
let parent = home
.join("runs")
.join(&state.id)
.join("artifacts/source-evidence");
let artifacts = parent.parent().context("artifacts")?;
plain(artifacts.parent().context("run")?)?;
create_plain_dir(artifacts)?;
create_plain_dir(&parent)?;
sync_directory(parent.parent().context("artifacts")?)?;
sync_directory(parent.parent().and_then(Path::parent).context("run")?)?;
let dir = parent.join(format!("{head}-{}", state.base_commit));
if dir.try_exists()? {
validate_snapshot(&dir).await?;
let manifest: Snapshot =
serde_json::from_slice(&std::fs::read(dir.join("manifest.json"))?)?;
ensure!(
manifest.head == head && manifest.base == state.base_commit,
"snapshot identity mismatch"
);
return Ok(dir);
}
let base_tree = git::rev_parse(&state.repo, &format!("{}^{{tree}}", state.base_commit)).await?;
let head_tree = git::rev_parse(&state.repo, &format!("{head}^{{tree}}")).await?;
let mut objects = BTreeSet::from([base_tree.clone(), head_tree.clone()]);
for tree in [&base_tree, &head_tree] {
let entries = capture(&state.repo, &["ls-tree", "-r", "-t", "-z", tree]).await?;
for entry in entries.split(|b| *b == 0).filter(|e| !e.is_empty()) {
let header =
std::str::from_utf8(entry.split(|b| *b == b'\t').next().context("tree entry")?)?;
let fields: Vec<_> = header.split_whitespace().collect();
ensure!(
fields.len() == 3 && matches!(fields[1], "tree" | "blob"),
"unsupported snapshot object"
);
objects.insert(fields[2].to_owned());
ensure!(objects.len() <= MAX_FILES, "snapshot object limit exceeded");
}
}
let anchor = if is_ancestor(&state.repo, &state.base_commit, head).await? {
retained_anchor(&state.repo, &state.base_commit, true).await?;
Some(state.base_commit.clone())
} else {
None
};
let commit_limit = format!("--max-count={}", MAX_FILES + 1);
let excluded = anchor.as_ref().map(|base| format!("^{base}"));
let mut revisions = vec![
"rev-list",
"--objects",
"--no-object-names",
&commit_limit,
head,
];
if let Some(excluded) = &excluded {
revisions.push(excluded);
}
let ancestry = capture(&state.repo, &revisions)
.await
.context("cannot enumerate snapshot commit ancestry")?;
for oid in std::str::from_utf8(&ancestry)?.lines() {
ensure!(
matches!(oid.len(), 40 | 64) && oid.bytes().all(|b| b.is_ascii_hexdigit()),
"invalid ancestry object identity"
);
objects.insert(oid.to_owned());
ensure!(
objects.len() <= MAX_FILES,
"snapshot ancestry/object limit exceeded"
);
}
let object_list = objects
.iter()
.map(|s| s.to_owned() + "\n")
.collect::<String>()
.into_bytes();
let sizes = input(
&state.repo,
&["cat-file", "--batch-check=%(objectsize)"],
object_list.clone(),
)
.await?;
let total =
std::str::from_utf8(&sizes)?
.lines()
.try_fold(0u64, |sum, size| -> Result<u64> {
let sum = sum
.checked_add(size.parse::<u64>()?)
.context("object size overflow")?;
ensure!(sum <= MAX_BYTES, "snapshot expanded object limit exceeded");
Ok(sum)
})?;
ensure!(
total <= MAX_BYTES,
"snapshot expanded object limit exceeded"
);
let pack = input(&state.repo, &["pack-objects", "--stdout"], object_list).await?;
let patch = source_patch(&state.repo, &state.base_commit, head).await?;
let index = source_index(&state.repo, &state.base_commit, head).await?;
ensure!(
pack.len() + patch.len() + index.len() <= MAX_BYTES.min(budget) as usize,
"snapshot storage limit exceeded"
);
let head_commit = capture(&state.repo, &["cat-file", "commit", head]).await?;
let base_commit = capture(&state.repo, &["cat-file", "commit", &state.base_commit]).await?;
ensure!(
head_commit.len() + base_commit.len() <= 1024 * 1024,
"snapshot commit metadata limit exceeded"
);
let manifest = Snapshot {
version: 2,
anchor,
head: head.to_owned(),
base: state.base_commit.clone(),
base_tree,
head_tree,
head_commit,
base_commit,
pack_hash: digest(&state.repo, pack.clone()).await?,
patch_hash: digest(&state.repo, patch.clone()).await?,
index_hash: digest(&state.repo, index.clone()).await?,
};
let manifest_bytes = serde_json::to_vec_pretty(&manifest)?;
ensure!(
pack.len() + patch.len() + index.len() + manifest_bytes.len()
<= MAX_BYTES.min(budget) as usize,
"snapshot publication budget exceeded"
);
let pending = dir.with_extension("pending");
let existing = snapshot_storage(&parent)?;
let staged = if pending.try_exists()? {
snapshot_storage(&pending)?
} else {
0
};
ensure!(
existing
.saturating_sub(staged)
.saturating_add((pack.len() + patch.len() + index.len() + manifest_bytes.len()) as u64)
<= MAX_BYTES,
"source evidence per-run storage limit exceeded"
);
create_plain_dir(&pending)?;
for entry in std::fs::read_dir(&pending)? {
let path = entry?.path();
plain(&path)?;
ensure!(
matches!(
path.file_name().and_then(|s| s.to_str()),
Some(
"objects.pack"
| "change.patch"
| "change.index"
| "manifest.json"
| "objects.pack.pending"
| "change.patch.pending"
| "change.index.pending"
| "manifest.json.pending"
)
),
"unknown pending archive content"
);
}
sync_directory(&parent)?;
let mut bytes = 0;
let mut files = 0;
store(&pending, "objects.pack", &pack, &mut bytes, &mut files)?;
store(&pending, "change.patch", &patch, &mut bytes, &mut files)?;
store(&pending, "change.index", &index, &mut bytes, &mut files)?;
store(
&pending,
"manifest.json",
&manifest_bytes,
&mut bytes,
&mut files,
)?;
validate_execution_snapshot_for(&pending, &state.repo).await?;
std::fs::rename(&pending, &dir)?;
sync_directory(&parent)?;
Ok(dir)
}
async fn load_snapshot(dir: &Path, repo: &Path) -> Result<Snapshot> {
plain(dir)?;
plain(&dir.join("manifest.json"))?;
ensure!(
std::fs::metadata(dir.join("manifest.json"))?.len() <= 8 * 1024 * 1024,
"snapshot manifest limit exceeded"
);
let manifest: Snapshot = serde_json::from_slice(&std::fs::read(dir.join("manifest.json"))?)?;
ensure!(
matches!(manifest.version, 1 | 2),
"unsupported snapshot version"
);
ensure!(
manifest
.anchor
.as_ref()
.is_none_or(|anchor| manifest.version == 2 && anchor == &manifest.base),
"invalid snapshot anchor"
);
for oid in [
&manifest.head,
&manifest.base,
&manifest.head_tree,
&manifest.base_tree,
] {
ensure!(
matches!(oid.len(), 40 | 64) && oid.bytes().all(|b| b.is_ascii_hexdigit()),
"invalid snapshot object ID"
);
}
for (oid, tree, commit) in [
(&manifest.head, &manifest.head_tree, &manifest.head_commit),
(&manifest.base, &manifest.base_tree, &manifest.base_commit),
] {
ensure!(
commit.starts_with(format!("tree {tree}\n").as_bytes()),
"snapshot commit/tree mismatch"
);
let actual = input(
repo,
&["hash-object", "-t", "commit", "--stdin"],
commit.clone(),
)
.await?;
ensure!(
std::str::from_utf8(&actual)?.trim() == oid,
"snapshot commit identity mismatch"
);
}
let mut total = 0;
for (name, hash) in [
("objects.pack", &manifest.pack_hash),
("change.patch", &manifest.patch_hash),
("change.index", &manifest.index_hash),
] {
let path = dir.join(name);
plain(&path)?;
total += std::fs::metadata(&path)?.len();
ensure!(total <= MAX_BYTES, "snapshot storage limit exceeded");
ensure!(
digest(repo, std::fs::read(path)?).await? == *hash,
"corrupt snapshot: {name}"
);
}
input(
repo,
&["unpack-objects", "-r"],
std::fs::read(dir.join("objects.pack"))?,
)
.await?;
for tree in [&manifest.base_tree, &manifest.head_tree] {
capture(repo, &["read-tree", tree]).await?;
ensure!(
git::git(repo, &["write-tree"]).await? == *tree,
"incomplete snapshot tree"
);
}
ensure!(
source_patch(repo, &manifest.base_tree, &manifest.head_tree).await?
== std::fs::read(dir.join("change.patch"))?,
"snapshot patch mismatch"
);
let index = source_index(repo, &manifest.base_tree, &manifest.head_tree).await?;
ensure!(
index == std::fs::read(dir.join("change.index"))?,
"snapshot index mismatch"
);
Ok(manifest)
}
pub(crate) async fn validate_identity(dir: &Path, head: &str, base: &str) -> Result<()> {
validate_snapshot(dir).await?;
let manifest: Snapshot = serde_json::from_slice(&std::fs::read(dir.join("manifest.json"))?)?;
ensure!(
manifest.head == head && manifest.base == base,
"snapshot identity mismatch"
);
Ok(())
}
async fn init_inspection(repo: &Path) -> Result<()> {
git::git(repo, &["init", "--quiet", "--template="]).await?;
let top = git::git(repo, &["rev-parse", "--show-toplevel"]).await?;
ensure!(
git::canonical_output_path(&top)? == repo,
"inspection worktree redirected by Git configuration"
);
git::git(repo, &["config", "core.autocrlf", "false"]).await?;
git::git(repo, &["config", "core.eol", "lf"]).await?;
Ok(())
}
pub(crate) async fn validate_snapshot(dir: &Path) -> Result<()> {
let temp = Inspection::new(&std::env::temp_dir())?;
let repo = temp.0.clone();
init_inspection(&repo).await?;
let manifest = load_snapshot(dir, &repo).await?;
capture(&repo, &["read-tree", &manifest.base_tree]).await?;
let patch = std::fs::read(dir.join("change.patch"))?;
if !patch.is_empty() {
input(&repo, &["apply", "--cached", "--binary", "-"], patch).await?;
}
ensure!(
git::git(&repo, &["write-tree"]).await? == manifest.head_tree,
"archived patch does not reconstruct the source tree"
);
capture(&repo, &["checkout-index", "--all", "--force"]).await?;
for tree in [&manifest.head_tree, &manifest.base_tree] {
capture(&repo, &["read-tree", "--reset", "-u", tree]).await?;
super::retention::tracked_base(&repo, tree).await?;
}
Ok(())
}
async fn retained_anchor(repo: &Path, base: &str, create: bool) -> Result<()> {
ensure!(
matches!(base.len(), 40 | 64) && base.bytes().all(|b| b.is_ascii_hexdigit()),
"invalid source anchor ID"
);
ensure!(
git::git(repo, &["rev-parse", "--is-shallow-repository"]).await? == "false",
"source anchor requires a complete shared repository"
);
let common = crate::repo_lock::common_dir(repo)
.await?
.context("missing shared repository")?;
let objects = common.join("objects");
plain(&objects).context("source anchor object store is not canonical")?;
for name in ["alternates", "http-alternates"] {
let path = objects.join("info").join(name);
match std::fs::symlink_metadata(&path) {
Ok(meta) => {
plain(&path)?;
ensure!(
meta.is_file() && meta.len() == 0,
"source anchor cannot depend on borrowed objects: {name}"
);
}
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
Err(e) => return Err(e.into()),
}
}
let promisor = git::git_raw(
repo,
&[
"config",
"--get-regexp",
r"^(extensions\.partialclone|remote\..*\.promisor)$",
],
)
.await?;
ensure!(
promisor.code == Some(1),
"source anchor cannot depend on promisor objects or unreadable configuration"
);
let packs = objects.join("pack");
if packs.try_exists()? {
plain(&packs)?;
for (count, entry) in std::fs::read_dir(&packs)?.enumerate() {
ensure!(
count < MAX_FILES,
"source anchor pack inspection limit exceeded"
);
ensure!(
!entry?.path().extension().is_some_and(|s| s == "promisor"),
"source anchor cannot depend on promisor packs"
);
}
}
let name = format!("refs/magi/source-anchors/{base}");
let mut file = common.clone();
for part in name.split('/') {
file.push(part);
match std::fs::symlink_metadata(&file) {
Ok(_) => plain(&file).context("noncanonical source anchor")?,
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
Err(e) => return Err(e.into()),
}
}
ensure!(
!git::git_raw(repo, &["symbolic-ref", "-q", &name])
.await?
.ok(),
"symbolic source anchor retained"
);
let existing = git::git_raw(repo, &["rev-parse", "--verify", &name]).await?;
if !existing.ok() {
ensure!(create, "retained source anchor missing: {name}");
capture(
repo,
&[
"fsck",
"--connectivity-only",
"--no-reflogs",
"--no-dangling",
base,
],
)
.await
.context("incomplete shared source anchor ancestry")?;
let absent = "0".repeat(base.len());
git::git(repo, &["update-ref", "--no-deref", &name, base, &absent]).await?;
}
ensure!(
git::rev_parse(repo, &name).await? == base,
"retained source anchor changed: {name}"
);
Ok(())
}
async fn inspection_anchor(repo: &Path, manifest: &Snapshot) -> Result<()> {
if let Some(anchor) = &manifest.anchor {
input(
repo,
&["hash-object", "-w", "-t", "commit", "--stdin"],
manifest.base_commit.clone(),
)
.await?;
std::fs::write(repo.join(".git/shallow"), format!("{anchor}\n"))?;
}
Ok(())
}
pub(crate) async fn validate_execution_snapshot_for(dir: &Path, repo: &Path) -> Result<()> {
validate_execution_snapshot(dir).await?;
let manifest: Snapshot = serde_json::from_slice(&std::fs::read(dir.join("manifest.json"))?)?;
if let Some(anchor) = &manifest.anchor {
retained_anchor(repo, anchor, false).await?;
}
Ok(())
}
pub(crate) async fn validate_execution_snapshot(dir: &Path) -> Result<()> {
validate_snapshot(dir).await?;
let temp = Inspection::new(&std::env::temp_dir())?;
init_inspection(&temp.0).await?;
let manifest = load_snapshot(dir, &temp.0).await?;
inspection_anchor(&temp.0, &manifest).await?;
input(
&temp.0,
&["hash-object", "-w", "-t", "commit", "--stdin"],
manifest.head_commit,
)
.await?;
capture(
&temp.0,
&[
"fsck",
"--connectivity-only",
"--no-reflogs",
"--no-dangling",
&manifest.head,
],
)
.await
.context("incomplete archived commit ancestry")?;
Ok(())
}
pub async fn restore_snapshot(dir: &Path, destination: &Path) -> Result<()> {
ensure!(
!destination.try_exists()?,
"restore destination already exists"
);
validate_snapshot(dir).await?;
let parent = destination
.parent()
.filter(|p| !p.as_os_str().is_empty())
.unwrap_or_else(|| Path::new("."))
.canonicalize()?;
let destination = parent.join(destination.file_name().context("restore directory name")?);
ensure!(
matches!(std::fs::symlink_metadata(&destination), Err(e) if e.kind() == std::io::ErrorKind::NotFound),
"restore destination already exists or is inaccessible"
);
let temp = Inspection::new(&parent)?;
let repo = temp.0.clone();
init_inspection(&repo).await?;
let manifest = load_snapshot(dir, &repo).await?;
capture(&repo, &["checkout-index", "--all"]).await?;
super::retention::tracked_base(&repo, &manifest.head_tree).await?;
let executable = match validate_execution_snapshot(dir).await {
Ok(()) => true,
Err(error) if manifest.version == 2 => return Err(error),
Err(_) => false,
};
if executable {
inspection_anchor(&repo, &manifest).await?;
input(
&repo,
&["hash-object", "-w", "-t", "commit", "--stdin"],
manifest.head_commit,
)
.await?;
capture(&repo, &["update-ref", "--no-deref", "HEAD", &manifest.head]).await?;
}
std::fs::rename(&temp.0, destination)?;
sync_directory(&parent)?;
Ok(())
}
pub(crate) async fn import_snapshot(dir: &Path, repo: &Path) -> Result<()> {
validate_execution_snapshot_for(dir, repo).await?;
let manifest: Snapshot = serde_json::from_slice(&std::fs::read(dir.join("manifest.json"))?)?;
if let Some(anchor) = &manifest.anchor {
capture(
repo,
&[
"fsck",
"--connectivity-only",
"--no-reflogs",
"--no-dangling",
anchor,
],
)
.await
.context("incomplete shared source anchor ancestry")?;
}
input(
repo,
&["unpack-objects", "-r"],
std::fs::read(dir.join("objects.pack"))?,
)
.await?;
input(
repo,
&["hash-object", "-w", "-t", "commit", "--stdin"],
manifest.head_commit,
)
.await?;
Ok(())
}
struct Inspection(std::path::PathBuf);
impl Inspection {
fn new(parent: &Path) -> Result<Self> {
for name in [
"GIT_DIR",
"GIT_WORK_TREE",
"GIT_COMMON_DIR",
"GIT_INDEX_FILE",
"GIT_OBJECT_DIRECTORY",
"GIT_ALTERNATE_OBJECT_DIRECTORIES",
"GIT_CONFIG_PARAMETERS",
"GIT_CONFIG_COUNT",
"GIT_CONFIG",
] {
ensure!(
std::env::var_os(name).is_none(),
"inspection refuses Git environment override: {name}"
);
}
Self::create_with(parent, crate::rng::entropy)
}
fn create_with(parent: &Path, mut next: impl FnMut() -> u64) -> Result<Self> {
let parent = parent.canonicalize()?;
for _ in 0..128 {
let path = parent.join(format!("magi-source-inspection-{:016x}", next()));
match std::fs::create_dir(&path) {
Ok(()) => return Ok(Self(path)),
Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => continue,
Err(error) => return Err(error).context("create source inspection directory"),
}
}
anyhow::bail!("source inspection directory name collision limit exceeded")
}
}
impl Drop for Inspection {
fn drop(&mut self) {
let _ = std::fs::remove_dir_all(&self.0);
}
}
fn create_plain_dir(path: &Path) -> Result<()> {
match std::fs::symlink_metadata(path) {
Ok(_) => plain(path)?,
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
plain(path.parent().context("evidence parent")?)?;
std::fs::create_dir(path)?;
}
Err(e) => return Err(e.into()),
}
ensure!(
std::fs::symlink_metadata(path)?.is_dir(),
"evidence directory is not a directory"
);
plain(path)
}
pub(crate) fn snapshot_storage(root: &Path) -> Result<u64> {
plain(root)?;
let mut dirs = vec![(root.to_owned(), 0usize)];
let mut files = 0usize;
let mut bytes = 0u64;
let mut entries = 0usize;
while let Some((dir, depth)) = dirs.pop() {
for entry in std::fs::read_dir(dir)? {
let path = entry?.path();
plain(&path)?;
entries += 1;
ensure!(entries <= MAX_FILES, "source evidence entry limit exceeded");
let meta = std::fs::symlink_metadata(&path)?;
if meta.is_dir() {
ensure!(
depth == 0 && dirs.len() < MAX_HEADS,
"unknown nested source evidence directory"
);
dirs.push((path, depth + 1));
} else {
files += 1;
bytes = bytes.saturating_add(meta.len());
ensure!(
files <= MAX_FILES && bytes <= MAX_BYTES,
"source evidence storage limit exceeded"
);
}
}
}
Ok(bytes)
}
#[cfg(all(test, unix))]
pub(crate) mod fixtures {
use super::*;
pub(crate) async fn shared_history(repo: &Path, parent: &str, count: usize) {
let mut stream = String::new();
for i in 1..=count {
let from = if i == 1 {
parent.to_owned()
} else {
format!(":{}", i - 1)
};
stream.push_str(&format!("commit refs/heads/main\nmark :{i}\ncommitter Fixture <fixture@example.invalid> {i} +0000\ndata 1\nx\nfrom {from}\n\n"));
}
input(repo, &["fast-import", "--quiet"], stream.into_bytes())
.await
.unwrap();
}
pub(crate) async fn tree_only_archive(
state: &RunState,
home: &Path,
head: &str,
) -> std::path::PathBuf {
let archive = snapshot(state, home, head, MAX_BYTES).await.unwrap();
let mut manifest: Snapshot =
serde_json::from_slice(&std::fs::read(archive.join("manifest.json")).unwrap()).unwrap();
manifest.version = 1;
manifest.anchor = None;
let mut objects = BTreeSet::from([manifest.base_tree.clone(), manifest.head_tree.clone()]);
for tree in [&manifest.base_tree, &manifest.head_tree] {
for oid in git::git(
&state.repo,
&["ls-tree", "-r", "-t", "--format=%(objectname)", tree],
)
.await
.unwrap()
.lines()
{
objects.insert(oid.to_owned());
}
}
let pack = input(
&state.repo,
&["pack-objects", "--stdout"],
objects
.iter()
.map(|s| s.to_owned() + "\n")
.collect::<String>()
.into_bytes(),
)
.await
.unwrap();
manifest.pack_hash = digest(&state.repo, pack.clone()).await.unwrap();
std::fs::write(archive.join("objects.pack"), pack).unwrap();
std::fs::write(
archive.join("manifest.json"),
serde_json::to_vec_pretty(&manifest).unwrap(),
)
.unwrap();
validate_snapshot(&archive).await.unwrap();
archive
}
}