//! Git transport for a step's `source_context` (R971-F5, W363 §6).
//!
//! The coordinator half ([`snapshot`] + [`push_snapshot`]) turns the camp's
//! live tree under the step's declared paths into a snapshot commit (chained
//! on the previous snapshot, see [`chain_parent`]) inside a
//! camp-side *staging* repo and pushes it to `refs/qed/<run>/<step>` on the
//! fleet git remote. The worker half ([`fetch_checkout`]) fetches that ref into
//! a node-persistent bare cache repo and checks it out into the step's
//! workspace. Both halves shell out to the host's `git` on `PATH` — never a
//! vendored or Alpine-specific one — so a glibc worker works the same as the
//! camp Mac.
//!
//! The staging repo is bare, with `objects/info/alternates` pointing at the
//! camp's own `.git/objects`, and every command runs with a `GIT_INDEX_FILE`
//! of the staging repo's own (a temp one, or a whole-tree snapshot's
//! [`KeptIndex`]): **nothing is ever written to the camp's index, objects or
//! refs.** Untracked-but-not-ignored files under the paths are included
//! (`git add -A`), which is what closes R855-B1 on this path.
//!
//! Tokens only ever travel in the environment (`GIT_CONFIG_*`), passed in by
//! the caller as ready-made pairs — this module never sees how they were
//! minted and never puts one in argv.
//!
//! @arch:see(.yah/docs/working/W363-fleet-git-remote.md)
use std::path::{Path, PathBuf};
use std::process::{Command, Output};
use std::time::{Duration, SystemTime};
use crate::runner::RunnerError;
/// Env var the step receives its fetch token in (W363 §6.4). Read by the
/// worker half; never set on the tar path.
pub const SOURCE_GIT_TOKEN_ENV: &str = "YAH_SOURCE_GIT_TOKEN";
/// `tar` forces today's tarball transport; anything else (or unset) means git
/// first with an automatic tar fallback when the push fails (operator call,
/// form F4371).
/// Reserved pipeline param (`--param source_commit=<sha>`): publish this
/// PRE-MADE commit, already present in the publisher's staging repo, as the
/// run's source instead of snapshotting the camp tree (R978-F5: the landing
/// gate checks the lander's candidate, not the live tree).
pub const SOURCE_COMMIT_PARAM: &str = "source_commit";
pub const SOURCE_TRANSPORT_ENV: &str = "YAH_QED_SOURCE_TRANSPORT";
/// URL scheme prefix that marks a git source context in
/// [`SOURCE_CONTEXT_URL_ENV`](crate::build_context::SOURCE_CONTEXT_URL_ENV).
pub const GIT_URL_PREFIX: &str = "git+";
/// Age past which local `refs/qed/*` (staging) and `refs/remotes/fleet/qed/*`
/// (worker cache) are pruned — the same 7 days the server's GC uses.
pub const REF_MAX_AGE: Duration = Duration::from_secs(7 * 24 * 3600);
/// True when the operator forced the tar transport.
pub fn tar_forced() -> bool {
std::env::var(SOURCE_TRANSPORT_ENV).is_ok_and(|v| v.trim().eq_ignore_ascii_case("tar"))
}
/// `refs/qed/<run>/<step>`, with both components reduced to ref-safe bytes.
pub fn run_ref(run_id: &str, step: &str) -> String {
format!("refs/qed/{}/{}", ref_component(run_id), ref_component(step))
}
fn ref_component(s: &str) -> String {
let out: String = s
.chars()
.map(|c| if c.is_ascii_alphanumeric() || matches!(c, '-' | '_') { c } else { '-' })
.collect();
if out.is_empty() { "_".into() } else { out }
}
/// `git+<remote>#<ref>` — the value `$YAH_SOURCE_CONTEXT_URL` carries on the
/// git path.
pub fn context_url(remote: &str, git_ref: &str) -> String {
format!("{GIT_URL_PREFIX}{remote}#{git_ref}")
}
/// Inverse of [`context_url`]: `(remote, ref)`, or `None` for a tar URL.
pub fn parse_context_url(url: &str) -> Option<(&str, &str)> {
url.strip_prefix(GIT_URL_PREFIX)?.split_once('#')
}
/// The coordinator's git transport for `source_context`, wired by the
/// embedder (the `yah` CLI mints fleet-git tokens; qed knows nothing of
/// cheers). The runner tries this first and falls back to the tar
/// [`BuildContextPublisher`](crate::build_context::BuildContextPublisher) on
/// any error.
#[async_trait::async_trait]
pub trait SourceGitPublisher: Send + Sync {
/// Snapshot `paths` and push it to `git_ref`, returning what the step's
/// env needs. `ttl` bounds the fetch token (the step timeout, else 6 h).
/// `commit` (the [`SOURCE_COMMIT_PARAM`] param) overrides the snapshot: push
/// that existing commit under `git_ref` as is.
async fn publish(
&self,
camp_root: &Path,
paths: &[PathBuf],
commit: Option<&str>,
git_ref: &str,
ttl: Duration,
) -> Result<GitPublished, RunnerError>;
}
/// A pushed snapshot: `url` for [`SOURCE_CONTEXT_URL_ENV`](crate::build_context::SOURCE_CONTEXT_URL_ENV),
/// `token` for [`SOURCE_GIT_TOKEN_ENV`].
#[derive(Clone)]
pub struct GitPublished {
pub url: String,
pub token: String,
pub snapshot: Snapshot,
}
/// Default fetch-token TTL when the step declares no timeout (W363 §5).
pub const DEFAULT_TOKEN_TTL: Duration = Duration::from_secs(6 * 3600);
/// A snapshot commit sitting in the staging repo.
#[derive(Debug, Clone)]
pub struct Snapshot {
pub commit: String,
/// The camp commit it was taken against (`HEAD` at snapshot time).
pub base: String,
/// Whether the tree under the paths differs from `base`'s.
pub dirty: bool,
}
/// Ensure the bare staging repo exists at `staging` with alternates pointing
/// at `camp_root`'s object store. Idempotent.
pub fn ensure_staging(camp_root: &Path, staging: &Path) -> Result<(), RunnerError> {
if !staging.join("HEAD").exists() {
std::fs::create_dir_all(staging).map_err(|e| err(format!("creating {}: {e}", staging.display())))?;
check(git_raw(None).arg("init").arg("--bare").arg("-q").arg(staging).output(), "git init --bare")?;
}
let objects = git_dir_of(camp_root)?.join("objects");
let alt = staging.join("objects/info/alternates");
let want = format!("{}\n", objects.display());
if std::fs::read_to_string(&alt).ok().as_deref() != Some(want.as_str()) {
std::fs::create_dir_all(alt.parent().unwrap())
.and_then(|_| std::fs::write(&alt, want))
.map_err(|e| err(format!("writing {}: {e}", alt.display())))?;
}
Ok(())
}
/// What a snapshot covers.
#[derive(Debug, Clone, Copy)]
pub enum SnapshotScope<'a> {
/// The step's `source_context` paths (camp-root-relative).
Paths(&'a [PathBuf]),
/// The whole tree: index seeded with `HEAD`'s tree, then
/// `git add -A` with no pathspec (untracked-not-ignored files ride,
/// tracked-but-ignored files stay, `.gitignore`d `target/` stays out),
/// minus `exclude`.
///
/// **`exclude` must not name tracked paths for the camp snapshot**: W366 §4
/// squashes `tree(camp_tip)` onto `main`, so an excluded tracked path would
/// be deleted from `main`. Only relay snapshots (W365) may exclude tracked
/// paths such as `.yah/db/*.turso`.
///
/// Each work tree's index is kept in the staging repo between snapshots
/// ([`KeptIndex`]), so a re-snapshot hashes only what changed.
WholeTree { exclude: &'a [PathBuf] },
}
/// Which commit a snapshot is chained on (its single parent).
#[derive(Debug, Clone, Copy)]
pub enum ChainParent<'a> {
/// Newest snapshot under the staging repo's `refs/qed/*`, restarted
/// parentless past [`CHAIN_MAX_AGE`] (see [`chain_parent`]).
NewestQedRef,
/// The tip of this ref in the staging repo (`refs/heads/camp` for the
/// lander, `refs/relays/<R>/tip` for W365); a missing ref seeds a root.
Named(&'a str),
/// Parentless root seed.
None,
}
/// Build the snapshot commit of `scope` in `staging`, chained per `chain`.
/// See the module docs for why nothing touches the camp's `.git`.
///
/// @yah:ticket(R978-T1, "Whole-tree snapshot variant: `snapshot` takes a scope (source_context | whole tree minus an exclusion list) and a caller-chosen chain parent")
/// @yah:status(review)
/// @yah:at(2026-10-13T08:37:16Z)
/// @yah:assignee(agent:bundle-anthropic-miravel)
/// @yah:phase(P1)
/// @yah:parent(R978)
/// @yah:next("Change `snapshot`'s signature, no _v2 (CLAUDE.md pre-1.0 rule). Scope: the step's source_context paths (today) or the whole tree minus an exclusion list. Chain parent: newest refs/qed/* (QED, today's `chain_parent`), a named ref (refs/heads/camp for the lander, refs/relays/<R>/tip for W365), or none (root seed). Fix the one caller, FleetGitSourcePublisher at app/yah/cli/src/qed_build_context.rs:269.")
/// @yah:next("Whole tree = index seeded from `git ls-tree -r <camp-commit>` plus `git add -A` with no pathspec, minus the exclusion list: tracked-but-ignored files ride, target/ stays out via .gitignore (W363 §6 recipe, W365 §2 variant).")
/// @yah:next("Regression test: a one-file edit after a chained whole-tree snapshot pushes one tree delta, not the whole tree (W363 §6's 7/7-objects measurement).")
/// @yah:verify("cargo test --manifest-path oss/qed/Cargo.toml -p yah-qed source_git")
/// @yah:gotcha("Excluding a TRACKED path from the camp snapshot deletes it from main: W366 §4 squashes tree(camp_tip) onto main. Relay snapshots (W365) may exclude .yah/db/*.turso; the camp snapshot must not, and waits on R975-T1 instead (inference from W366 §4 + R975).")
/// @yah:gotcha("Shared with W365: relay snapshots chain on refs/relays/<R>/tip through this same function, so the chain-parent parameter is the W365 seam.")
/// @yah:files(oss/qed/crates/qed/src/source_git.rs)
/// @yah:files(app/yah/cli/src/qed_build_context.rs)
/// @yah:handoff("snapshot() in oss/qed/crates/qed/src/source_git.rs now takes (camp_root, staging, SnapshotScope, ChainParent). SnapshotScope::{Paths(&[PathBuf]), WholeTree{exclude}}; ChainParent::{NewestQedRef, Named(ref), None}. WholeTree = ls-tree -r HEAD seeded index + pathspec-less add -A, then git rm --cached on the exclude list. Named(ref) missing ref seeds a root. Caller FleetGitSourcePublisher (app/yah/cli/src/qed_build_context.rs) uses Paths + NewestQedRef. The exclude doc warns that the camp snapshot must not exclude tracked paths (W366 §4 squash would delete them from main).")
/// @yah:verify("cargo test -p yah-qed source_git: 10 passed, 0 failed (2 new: whole_tree_snapshot_honours_ignore_and_exclusions, chained_whole_tree_snapshot_pushes_only_the_delta, which asserts a one-file edit moves <=5 new objects). cargo check -p yah: one error, a peer's, at app/yah/cli/src/camp.rs:11652 (E0382 borrow of moved `retry_of`); no errors in qed_build_context.rs. An earlier yah-qed run was red on a peer's runner.rs (ElectRequest.prefer_node, deploy signature), green on rerun.")
/// @yah:handoff("snapshot() now takes (camp_root, staging, SnapshotScope, ChainParent). Scope: Paths | WholeTree{exclude}; chain: NewestQedRef | Named(ref) | None. Caller FleetGitSourcePublisher uses Paths + NewestQedRef. exclude doc warns camp snapshot must not exclude tracked paths.")
/// @yah:verify("cargo test yah-qed source_git: 10 pass, 0 fail (2 new incl. one-file-edit <=5 objects). cargo check -p yah: only error is peer's camp.rs:11652 E0382 retry_of; none in qed_build_context.rs.")
/// @yah:handoff("snapshot() now takes (camp_root, staging, SnapshotScope, ChainParent). Scope: Paths | WholeTree{exclude}; chain: NewestQedRef | Named(ref) | None. Caller FleetGitSourcePublisher uses Paths + NewestQedRef. exclude doc warns camp snapshot must not exclude tracked paths.")
/// @yah:verify("cargo test yah-qed source_git: 10 pass, 0 fail (2 new incl. one-file-edit <=5 objects). cargo check -p yah: only error is peer's camp.rs:11652 E0382 retry_of; none in qed_build_context.rs.")
/// @yah:handoff("snapshot() now takes (camp_root, staging, SnapshotScope, ChainParent). Scope: Paths | WholeTree{exclude}; chain: NewestQedRef | Named(ref) | None. Caller FleetGitSourcePublisher uses Paths + NewestQedRef. exclude doc warns camp snapshot must not exclude tracked paths.")
/// @yah:verify("cargo test yah-qed source_git: 10 pass, 0 fail (2 new incl. one-file-edit <=5 objects). cargo check -p yah: only error is peer's camp.rs:11652 E0382 retry_of; none in qed_build_context.rs.")
pub fn snapshot(
camp_root: &Path,
staging: &Path,
scope: SnapshotScope<'_>,
chain: ChainParent<'_>,
) -> Result<Snapshot, RunnerError> {
let paths: &[PathBuf] = match scope {
SnapshotScope::Paths(p) => {
if p.is_empty() {
return Err(err("source context requested with no paths".into()));
}
p
}
SnapshotScope::WholeTree { .. } => &[],
};
ensure_staging(camp_root, staging)?;
let base = stdout(
git_raw(None).arg("-C").arg(camp_root).args(["rev-parse", "HEAD^{commit}"]).output(),
"git rev-parse HEAD",
)?;
let kept = match scope {
SnapshotScope::WholeTree { .. } => KeptIndex::acquire(staging, camp_root),
SnapshotScope::Paths(_) => None,
};
let (index_path, _cleanup) = match &kept {
Some(k) => (k.path.clone(), None),
None => {
let index = tempfile::NamedTempFile::new_in(staging).map_err(|e| err(format!("temp index: {e}")))?;
let index_path = index.path().to_path_buf();
// git refuses an existing zero-byte file as an index.
drop(index);
(index_path.clone(), Some(RemoveOnDrop(index_path)))
}
};
let in_tree = |cmd: &mut Command| {
cmd.env("GIT_DIR", staging)
.env("GIT_WORK_TREE", camp_root)
.env("GIT_INDEX_FILE", &index_path)
.current_dir(camp_root);
};
// The index starts as exactly `base`'s tree under the scope, tracked-but-
// ignored files included. Whole tree: a one-tree `read-tree -m` keeps a kept index's
// stat data for every path whose blob `base` still has, so `add -A` below
// re-hashes only what changed. `-i`: never consult the work tree here.
let listing = match scope {
SnapshotScope::WholeTree { .. } => {
let read_tree = || {
let mut rt = git_raw(None);
in_tree(&mut rt);
rt.args(["read-tree", "-m", "-i", &base]);
check(rt.output(), "git read-tree -m -i")
};
if let Err(e) = read_tree() {
if kept.is_none() {
return Err(e);
}
// A kept index is a cache: one git cannot read is dropped, never fatal.
let _ = std::fs::remove_file(&index_path);
read_tree()?;
}
None
}
SnapshotScope::Paths(_) => {
let mut ls = git_raw(None);
in_tree(&mut ls);
ls.args(["ls-tree", "-r", "-z", &base, "--"]).args(paths);
let listing = check(ls.output(), "git ls-tree")?.stdout;
let mut upd = git_raw(None);
in_tree(&mut upd);
upd.args(["update-index", "-z", "--index-info"]);
run_with_stdin(upd, &listing, "git update-index --index-info")?;
Some(listing)
}
};
let mut add = git_raw(None);
in_tree(&mut add);
// `--ignore-errors` is NOT passed: a file that cannot be read must fail the
// snapshot rather than ship a tree that silently lacks it.
// Whole tree: no pathspec at all, so the index seeded from `base` keeps
// tracked-but-ignored files and `.gitignore` keeps `target/` out.
add.args(["add", "-A"]);
if !paths.is_empty() {
add.arg("--").args(paths);
}
check(add.output(), "git add -A")?;
if let SnapshotScope::WholeTree { exclude } = scope {
if !exclude.is_empty() {
let mut rm = git_raw(None);
in_tree(&mut rm);
rm.args(["rm", "-r", "--cached", "-q", "--ignore-unmatch", "--"]).args(exclude);
check(rm.output(), "git rm --cached (exclude)")?;
}
}
let mut wt = git_raw(None);
in_tree(&mut wt);
wt.arg("write-tree");
let tree = stdout(wt.output(), "git write-tree")?;
if stdout(git_raw(Some(staging)).args(["ls-tree", &tree]).output(), "git ls-tree <tree>")?.is_empty() {
return Err(err(format!(
"`source_context` {paths:?} matched no files — the step would fetch an empty \
source tree. Check the paths are camp-root-relative."
)));
}
// The same paths, as committed: compare to decide `dirty`.
let base_tree = match listing {
None => stdout(
git_raw(Some(staging)).args(["rev-parse", &format!("{base}^{{tree}}")]).output(),
"git rev-parse <base tree>",
)?,
Some(listing) => {
let mut c = git_raw(None);
in_tree(&mut c);
let mut idx = git_raw(None);
let tmp2 = tempfile::NamedTempFile::new_in(staging).map_err(|e| err(format!("temp index: {e}")))?;
let p2 = tmp2.path().to_path_buf();
drop(tmp2);
let _c2 = RemoveOnDrop(p2.clone());
idx.env("GIT_DIR", staging).env("GIT_INDEX_FILE", &p2).args(["update-index", "-z", "--index-info"]);
run_with_stdin(idx, &listing, "git update-index (base)")?;
c.env("GIT_INDEX_FILE", &p2).arg("write-tree");
stdout(c.output(), "git write-tree (base)")?
}
};
let dirty = base_tree != tree;
let mut ct = git_raw(Some(staging));
ct.env("GIT_AUTHOR_NAME", "yah-qed")
.env("GIT_AUTHOR_EMAIL", "qed@yah.invalid")
.env("GIT_COMMITTER_NAME", "yah-qed")
.env("GIT_COMMITTER_EMAIL", "qed@yah.invalid")
.args(["commit-tree", &tree, "-m", &format!("yah-snapshot-of: {base} dirty={dirty}")]);
let parent = match chain {
ChainParent::NewestQedRef => chain_parent(staging),
ChainParent::Named(r) => stdout(
git_raw(Some(staging)).args(["rev-parse", "--verify", "--quiet", &format!("{r}^{{commit}}")]).output(),
"git rev-parse <chain ref>",
)
.ok()
.filter(|c| !c.is_empty()),
ChainParent::None => None,
};
if let Some(parent) = parent {
ct.args(["-p", &parent]);
}
let commit = stdout(ct.output(), "git commit-tree")?;
Ok(Snapshot { commit, base, dirty })
}
/// Resolve a PRE-MADE commit already in `staging` as the run's [`Snapshot`]
/// (the [`SOURCE_COMMIT_PARAM`] path): nothing is built, only verified present.
pub fn existing_commit(staging: &Path, commit: &str) -> Result<Snapshot, RunnerError> {
let full = stdout(
git_raw(Some(staging)).args(["rev-parse", "--verify", "--quiet", &format!("{commit}^{{commit}}")]).output(),
"git rev-parse <source_commit>",
)
.ok()
.filter(|c| !c.is_empty())
.ok_or_else(|| err(format!("`{SOURCE_COMMIT_PARAM}` {commit} is not a commit in {}", staging.display())))?;
Ok(Snapshot { commit: full.clone(), base: full, dirty: false })
}
/// Age past which a snapshot chain is restarted parentless, so no chain spans
/// more than this and the 7-day ref GC can free a whole old chain.
const CHAIN_MAX_AGE: Duration = Duration::from_secs(6 * 24 * 3600);
/// The newest snapshot in `staging`'s `refs/qed/*`, if its chain's root is
/// younger than [`CHAIN_MAX_AGE`].
///
/// # Why snapshots are chained, not parentless (deviation from W363 §6.1)
///
/// `upload-pack` only excludes trees and blobs it can reach through the
/// *parents* of the commits it sends (`mark_edges_uninteresting`). A `have`
/// for an unrelated root commit excludes that commit alone, so with parentless
/// snapshots every fetch re-sent the full tree — measured by
/// `push_fetch_checkout_round_trip_moves_only_the_delta`: 7 of 7 objects for a
/// one-file edit. Parenting each snapshot on the previous one gives the server
/// an edge, and the first push of a chain still carries one tree.
fn chain_parent(staging: &Path) -> Option<String> {
let out = git_raw(Some(staging))
.args(["for-each-ref", "--sort=-committerdate", "--count=1", "--format=%(objectname)", "refs/qed/"])
.output()
.ok()?;
let tip = String::from_utf8_lossy(&out.stdout).trim().to_string();
if tip.is_empty() {
return None;
}
let roots = git_raw(Some(staging))
.args(["log", "--max-parents=0", "--format=%ct", &tip])
.output()
.ok()?;
let root_ts: u64 = String::from_utf8_lossy(&roots.stdout).lines().next()?.trim().parse().ok()?;
let now = SystemTime::now().duration_since(SystemTime::UNIX_EPOCH).ok()?.as_secs();
(now.saturating_sub(root_ts) < CHAIN_MAX_AGE.as_secs()).then_some(tip)
}
/// Push `commit` to `git_ref` on `remote`, then mirror the ref locally so the
/// next push's negotiation excludes what the remote already has. `auth_env`
/// is the caller's `GIT_CONFIG_*` triple.
pub fn push_snapshot(
staging: &Path,
remote: &str,
commit: &str,
git_ref: &str,
auth_env: &[(&str, String)],
) -> Result<(), RunnerError> {
let mut push = git_raw(Some(staging));
push.envs(auth_env.iter().map(|(k, v)| (*k, v.as_str())))
.args(["push", "--quiet", remote, &format!("+{commit}:{git_ref}")]);
check(push.output(), &format!("git push to {remote}"))?;
check(git_raw(Some(staging)).args(["update-ref", git_ref, commit]).output(), "git update-ref")?;
prune_refs(staging, "refs/qed/");
Ok(())
}
/// What a worker fetch moved, parsed from git's `--verbose` progress so a test
/// (and the step log) can see that a second snapshot transferred only the delta.
#[derive(Debug, Clone, Default)]
pub struct FetchStats {
pub oid: String,
/// Objects the server packed for us (`Total N` on stderr); `None` when the
/// fetch transferred nothing at all.
pub objects: Option<u64>,
/// git's fetch stderr, for the step log.
pub log: String,
}
/// The worker-side materializer, [`SOURCE_FETCH_ENV`]'s value. One POSIX `sh`
/// script for both transports; see the file for the contract.
pub const SOURCE_FETCH_SCRIPT: &str = include_str!("source_fetch.sh");
/// Env var qed hands [`SOURCE_FETCH_SCRIPT`] to a step in, next to
/// [`SOURCE_CONTEXT_URL_ENV`](crate::build_context::SOURCE_CONTEXT_URL_ENV). A
/// step materializes its source with `sh -c "$YAH_SOURCE_FETCH" yah-source-fetch <dest>`
/// rather than hand-rolling a fetch, so a new transport reaches every pipeline.
pub const SOURCE_FETCH_ENV: &str = "YAH_SOURCE_FETCH";
/// Env var that overrides where the script keeps its bare cache repo.
pub const SOURCE_GIT_CACHE_ENV: &str = "YAH_SOURCE_GIT_CACHE";
/// In-process run of [`SOURCE_FETCH_SCRIPT`]: materialize `context_url` (git+
/// or tar) into `workspace`, with `cache_repo` as the bare git cache. The
/// script is the one path a step runs too, so this is how tests exercise it.
///
/// A failure is the step's failure, and on the git path the message names the
/// tar override — guessing a fallback here would hide that the git path is down.
pub fn fetch_checkout(
cache_repo: &Path,
context_url: &str,
token: Option<&str>,
workspace: &Path,
) -> Result<FetchStats, RunnerError> {
let mut sh = Command::new("sh");
sh.arg("-c")
.arg(SOURCE_FETCH_SCRIPT)
.arg("yah-source-fetch")
.arg(workspace)
.env(crate::build_context::SOURCE_CONTEXT_URL_ENV, context_url)
.env(SOURCE_GIT_CACHE_ENV, cache_repo)
.env_remove(SOURCE_GIT_TOKEN_ENV);
if let Some(t) = token {
sh.env(SOURCE_GIT_TOKEN_ENV, t);
}
let out = sh.output().map_err(|e| err(format!("running yah-source-fetch: {e}")))?;
let log = String::from_utf8_lossy(&out.stderr).into_owned();
if !out.status.success() {
return Err(err(log.trim().to_string()));
}
let oid = log
.lines()
.find_map(|l| l.strip_prefix("yah-source-fetch: checked out "))
.and_then(|r| r.split_whitespace().next())
.unwrap_or_default()
.to_string();
Ok(FetchStats { oid, objects: parse_total_objects(&log), log })
}
/// The attach return path's worker half (R969-F7), [`SOURCE_SHIP_ENV`]'s
/// value. See the file for the contract.
pub const SOURCE_SHIP_SCRIPT: &str = include_str!("source_ship.sh");
/// Env var a `qed.attach` debug forge receives [`SOURCE_SHIP_SCRIPT`] in.
pub const SOURCE_SHIP_ENV: &str = "YAH_SOURCE_SHIP";
/// The snapshot commit the debug forge's workspace was checked out from.
pub const ATTACH_SNAPSHOT_ENV: &str = "YAH_ATTACH_SNAPSHOT";
/// The ref `yah-source-ship` pushes to ([`return_ref`]).
pub const RETURN_REF_ENV: &str = "YAH_RETURN_REF";
/// The debug forge's `git:write` token. Env only, never argv.
pub const RETURN_GIT_TOKEN_ENV: &str = "YAH_RETURN_GIT_TOKEN";
/// `refs/qed/<run>/<step>.return` — where an attach's edits come back.
///
/// A sibling of [`run_ref`], not a child: `refs/qed/<run>/<step>/return`
/// cannot coexist with the snapshot ref `refs/qed/<run>/<step>` (git refs are
/// paths, and one cannot be both a file and a directory). `.` never appears in
/// a [`run_ref`] component, so the suffix cannot collide with another step.
/// It stays under `refs/qed/**`, the only family a QED token may write.
pub fn return_ref(run_id: &str, step: &str) -> String {
format!("{}.return", run_ref(run_id, step))
}
/// In-process run of [`SOURCE_SHIP_SCRIPT`], the twin of [`fetch_checkout`]:
/// commit `workspace` onto `snapshot` in `cache_repo` and push it to
/// `ret_ref` on the context URL's remote. Returns the pushed commit.
pub fn ship_return(
cache_repo: &Path,
context_url: &str,
snapshot: &str,
ret_ref: &str,
token: &str,
workspace: &Path,
) -> Result<String, RunnerError> {
let out = Command::new("sh")
.arg("-c")
.arg(SOURCE_SHIP_SCRIPT)
.arg("yah-source-ship")
.arg(workspace)
.env(crate::build_context::SOURCE_CONTEXT_URL_ENV, context_url)
.env(SOURCE_GIT_CACHE_ENV, cache_repo)
.env(ATTACH_SNAPSHOT_ENV, snapshot)
.env(RETURN_REF_ENV, ret_ref)
.env(RETURN_GIT_TOKEN_ENV, token)
.output()
.map_err(|e| err(format!("running yah-source-ship: {e}")))?;
let log = String::from_utf8_lossy(&out.stderr).into_owned();
if !out.status.success() {
return Err(err(log.trim().to_string()));
}
Ok(log
.lines()
.find_map(|l| l.strip_prefix("yah-source-ship: pushed "))
.and_then(|r| r.split_whitespace().next())
.unwrap_or_default()
.to_string())
}
/// What [`harvest_return`] applied.
#[derive(Debug, Clone)]
pub struct Harvested {
/// The return commit fetched.
pub commit: String,
/// Camp-root-relative paths the diff touched; empty when nothing changed.
pub files: Vec<String>,
}
/// The attach return path's coordinator half (R969-F7): fetch `ret_ref` from
/// `remote` into the staging repo, then apply `git diff <snapshot>..<return>`
/// to `camp_root`'s working tree as **uncommitted** changes.
///
/// Refuses, changing nothing, when the return commit does not descend from
/// `snapshot` or when the patch does not apply cleanly (the camp tree moved
/// under the same lines since the snapshot). Like [`snapshot`], it never
/// writes the camp's index, objects or refs: git runs against the staging
/// repo with a scratch index, and only the working-tree files change.
pub fn harvest_return(
camp_root: &Path,
staging: &Path,
remote: &str,
ret_ref: &str,
snapshot: &str,
auth_env: &[(&str, String)],
) -> Result<Harvested, RunnerError> {
ensure_staging(camp_root, staging)?;
let mut fetch = git_raw(Some(staging));
fetch
.envs(auth_env.iter().map(|(k, v)| (*k, v.as_str())))
.args(["fetch", "--quiet", "--no-tags", remote, &format!("+{ret_ref}:{ret_ref}")]);
check(fetch.output(), &format!("git fetch {ret_ref} from {remote}"))?;
let commit = stdout(
git_raw(Some(staging)).args(["rev-parse", "--verify", &format!("{ret_ref}^{{commit}}")]).output(),
"git rev-parse <return>",
)?;
let descends = git_raw(Some(staging))
.args(["merge-base", "--is-ancestor", snapshot, &commit])
.status()
.map_err(|e| err(format!("git merge-base: {e}")))?;
if !descends.success() {
return Err(err(format!(
"{ret_ref} ({commit}) does not descend from the attach snapshot {snapshot}; refusing to harvest it"
)));
}
let files: Vec<String> = stdout(
git_raw(Some(staging)).args(["diff", "--name-only", "-z", snapshot, &commit]).output(),
"git diff --name-only",
)?
.split('\0')
.filter(|s| !s.is_empty())
.map(str::to_string)
.collect();
if files.is_empty() {
return Ok(Harvested { commit, files });
}
let patch = check(
git_raw(Some(staging)).args(["diff", "--binary", "--full-index", snapshot, &commit]).output(),
"git diff <snapshot>..<return>",
)?
.stdout;
let index = tempfile::NamedTempFile::new_in(staging).map_err(|e| err(format!("temp index: {e}")))?;
let index_path = index.path().to_path_buf();
drop(index);
let _cleanup = RemoveOnDrop(index_path.clone());
// `git apply` without `--index`/`--cached` writes only working-tree files;
// GIT_DIR + the scratch index make sure the camp's `.git` is not even read.
let apply = |check_only: bool| {
let mut c = git_raw(Some(staging));
c.env("GIT_WORK_TREE", camp_root).env("GIT_INDEX_FILE", &index_path).current_dir(camp_root);
c.arg("apply");
if check_only {
c.arg("--check");
}
c.arg("-");
c
};
run_with_stdin(apply(true), &patch, "git apply --check").map_err(|e| {
err(format!(
"the return diff does not apply to the camp tree, which changed under the same lines since the snapshot — nothing was written. {e}"
))
})?;
run_with_stdin(apply(false), &patch, "git apply")?;
Ok(Harvested { commit, files })
}
/// Pull `N` out of git's `Total N (delta …)` progress line.
fn parse_total_objects(stderr: &str) -> Option<u64> {
stderr
.split(['\n', '\r'])
.filter_map(|l| l.trim().split_once("Total ").map(|(_, r)| r))
.filter_map(|r| r.split_whitespace().next()?.trim_end_matches(',').parse().ok())
.last()
}
/// Best-effort: delete refs under `prefix` whose commit is older than
/// [`REF_MAX_AGE`], then `gc --auto`. Never fails the caller.
fn prune_refs(repo: &Path, prefix: &str) {
let Ok(out) = git_raw(Some(repo))
.args(["for-each-ref", "--format=%(refname) %(committerdate:unix)", prefix])
.output()
else {
return;
};
let now = SystemTime::now().duration_since(SystemTime::UNIX_EPOCH).map(|d| d.as_secs()).unwrap_or(0);
let mut pruned = false;
for line in String::from_utf8_lossy(&out.stdout).lines() {
let Some((name, ts)) = line.rsplit_once(' ') else { continue };
let Ok(ts) = ts.parse::<u64>() else { continue };
if now.saturating_sub(ts) > REF_MAX_AGE.as_secs() {
pruned |= git_raw(Some(repo)).args(["update-ref", "-d", name]).status().is_ok_and(|s| s.success());
}
}
if pruned {
let _ = git_raw(Some(repo)).args(["gc", "--auto", "--quiet"]).status();
}
}
fn git_dir_of(camp_root: &Path) -> Result<PathBuf, RunnerError> {
// The COMMON dir: objects live there even when `camp_root` is a linked
// worktree (whose own git dir has no objects/; R978-F10 lands a patch
// worktree's tree).
let dir = stdout(
git_raw(None).arg("-C").arg(camp_root).args(["rev-parse", "--path-format=absolute", "--git-common-dir"]).output(),
"git rev-parse --git-common-dir",
)?;
Ok(PathBuf::from(dir))
}
/// `git` from PATH with a hermetic config surface: no prompts, and `git_dir`
/// as `GIT_DIR` when given.
fn git_raw(git_dir: Option<&Path>) -> Command {
let mut c = Command::new("git");
c.env("GIT_TERMINAL_PROMPT", "0");
if let Some(d) = git_dir {
c.env("GIT_DIR", d);
}
c
}
fn run_with_stdin(mut cmd: Command, input: &[u8], what: &str) -> Result<Output, RunnerError> {
use std::io::Write;
use std::process::Stdio;
let mut child = cmd
.stdin(Stdio::piped())
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()
.map_err(|e| err(format!("{what}: {e}")))?;
child.stdin.take().unwrap().write_all(input).map_err(|e| err(format!("{what}: {e}")))?;
check(child.wait_with_output(), what)
}
fn check(out: std::io::Result<Output>, what: &str) -> Result<Output, RunnerError> {
let out = out.map_err(|e| err(format!("{what}: {e} (is git on PATH?)")))?;
if !out.status.success() {
return Err(err(format!("{what} failed: {}", String::from_utf8_lossy(&out.stderr).trim())));
}
Ok(out)
}
fn stdout(out: std::io::Result<Output>, what: &str) -> Result<String, RunnerError> {
Ok(String::from_utf8_lossy(&check(out, what)?.stdout).trim().to_string())
}
fn err(msg: String) -> RunnerError {
RunnerError::InvalidConfig(format!("source-context (git): {msg}"))
}
/// The index a whole-tree snapshot keeps for its work tree between runs,
/// under the staging repo's `snapshot-index/` and never in the camp's `.git`.
/// A fresh index seeded from `ls-tree` has no stat data, so `git add -A` read
/// and hashed every file of the tree on every snapshot. That was ~1.0 s of a
/// 1.2 s relay snapshot over 5.6k files and 153 MiB (R979-T20). Kept, it only
/// re-hashes paths whose stat changed or whose blob differs from the base.
///
/// Held under an exclusive lock for the whole snapshot. `acquire` is `None`
/// when another snapshot of the same tree holds it, or when the file cannot be
/// made. Either way the snapshot falls back to a temp index: the same tree, at
/// the full hashing cost.
struct KeptIndex {
path: PathBuf,
_lock: std::fs::File,
}
impl KeptIndex {
fn acquire(staging: &Path, worktree: &Path) -> Option<Self> {
use sha2::{Digest, Sha256};
let dir = staging.join("snapshot-index");
std::fs::create_dir_all(&dir).ok()?;
let tree = std::fs::canonicalize(worktree).unwrap_or_else(|_| worktree.to_path_buf());
let name = hex::encode(&Sha256::digest(tree.as_os_str().as_encoded_bytes())[..8]);
let lock = std::fs::OpenOptions::new()
.create(true)
.truncate(false)
.write(true)
.open(dir.join(format!("{name}.flock")))
.ok()?;
lock.try_lock().ok()?;
// Every writer of this index holds `lock`, so a git `index.lock` left
// beside it belongs to a run that died mid-write.
let _ = std::fs::remove_file(dir.join(format!("{name}.lock")));
// The sidecar is what lets a later acquire tell a retired worktree's
// entry from a live one. Best-effort, like the whole sweep.
let sidecar = dir.join(format!("{name}.worktree"));
if std::fs::read(&sidecar).ok().as_deref() != Some(tree.as_os_str().as_encoded_bytes()) {
let _ = std::fs::write(&sidecar, tree.as_os_str().as_encoded_bytes());
}
sweep_retired(&dir, &name, LEGACY_ENTRY_MAX_AGE);
Some(Self { path: dir.join(name), _lock: lock })
}
}
/// An entry with no `.worktree` sidecar (made before sidecars existed) cannot
/// be tied to a worktree; it is dropped once untouched this long. It is a
/// cache, so a wrong drop only costs a re-seed.
const LEGACY_ENTRY_MAX_AGE: std::time::Duration = std::time::Duration::from_secs(14 * 24 * 3600);
/// Removes sibling kept-index entries (R979-T20 cleanup) whose worktree no
/// longer exists, or legacy sidecar-less ones older than `legacy_max_age`.
/// Each entry's flock is taken with `try_lock` and held across its removal; a
/// held one is skipped, never waited on. Failures are ignored: the sweep must
/// never fail a snapshot.
fn sweep_retired(dir: &Path, own: &str, legacy_max_age: std::time::Duration) {
let Ok(rd) = std::fs::read_dir(dir) else { return };
let mut stems = std::collections::BTreeSet::new();
for e in rd.flatten() {
let n = e.file_name().to_string_lossy().into_owned();
if let Some(stem) = n.split('.').next().filter(|s| !s.is_empty() && *s != own) {
stems.insert(stem.to_string());
}
}
for stem in stems {
let Ok(lock) = std::fs::OpenOptions::new().write(true).open(dir.join(format!("{stem}.flock"))) else {
continue;
};
if lock.try_lock().is_err() {
continue;
}
let names = ["", ".lock", ".worktree"].map(|ext| dir.join(format!("{stem}{ext}")));
let retired = match std::fs::read(dir.join(format!("{stem}.worktree"))) {
Ok(b) => !{
#[cfg(unix)]
{
use std::os::unix::ffi::OsStrExt;
std::path::Path::new(std::ffi::OsStr::from_bytes(&b)).exists()
}
#[cfg(not(unix))]
{
Path::new(&*String::from_utf8_lossy(&b)).exists()
}
},
Err(_) => std::fs::metadata(dir.join(format!("{stem}.flock")))
.and_then(|m| m.modified())
.ok()
.and_then(|t| t.elapsed().ok())
.is_some_and(|age| age > legacy_max_age),
};
if retired {
for f in names {
let _ = std::fs::remove_file(f);
}
let _ = std::fs::remove_file(dir.join(format!("{stem}.flock")));
}
}
}
struct RemoveOnDrop(PathBuf);
impl Drop for RemoveOnDrop {
fn drop(&mut self) {
let _ = std::fs::remove_file(&self.0);
}
}
#[cfg(test)]
mod tests {
use super::*;
fn sh(dir: &Path, args: &[&str]) {
let out = Command::new("git")
.args(args)
.current_dir(dir)
.env("GIT_AUTHOR_NAME", "t")
.env("GIT_AUTHOR_EMAIL", "t@t")
.env("GIT_COMMITTER_NAME", "t")
.env("GIT_COMMITTER_EMAIL", "t@t")
.output()
.unwrap();
assert!(out.status.success(), "git {args:?}: {}", String::from_utf8_lossy(&out.stderr));
}
/// A camp with a committed `pkg/` and an unrelated `other/`.
fn camp() -> tempfile::TempDir {
let d = tempfile::tempdir().unwrap();
let p = d.path();
sh(p, &["init", "-q"]);
std::fs::create_dir_all(p.join("pkg/src")).unwrap();
std::fs::create_dir_all(p.join("other")).unwrap();
std::fs::write(p.join("pkg/Cargo.toml"), "[package]\n").unwrap();
std::fs::write(p.join("pkg/src/lib.rs"), "pub fn a() {}\n").unwrap();
std::fs::write(p.join("other/x"), "x\n").unwrap();
std::fs::write(p.join(".gitignore"), "target/\n").unwrap();
sh(p, &["add", "-A"]);
sh(p, &["commit", "-q", "-m", "init"]);
d
}
fn camp_git_state(p: &Path) -> (String, String, Vec<u8>) {
let o = |a: &[&str]| Command::new("git").args(a).current_dir(p).output().unwrap().stdout;
(
String::from_utf8(o(&["status", "--porcelain"])).unwrap(),
String::from_utf8(o(&["for-each-ref"])).unwrap(),
std::fs::read(p.join(".git/index")).unwrap(),
)
}
#[test]
fn snapshot_ships_untracked_files_and_leaves_the_camp_git_alone() {
let c = camp();
let p = c.path();
std::fs::write(p.join("pkg/src/new.rs"), "// untracked\n").unwrap();
std::fs::create_dir_all(p.join("pkg/target")).unwrap();
std::fs::write(p.join("pkg/target/junk"), "ignored").unwrap();
let before = camp_git_state(p);
// Outside the camp here; in a real camp it sits under the gitignored
// `.yah/cache/`, so it never shows in `git status` either.
let st = tempfile::tempdir().unwrap();
let staging = st.path().join("yah.git");
let s = snapshot(p, &staging, SnapshotScope::Paths(&[PathBuf::from("pkg")]), ChainParent::NewestQedRef).unwrap();
assert!(s.dirty);
let files = stdout(git_raw(Some(&staging)).args(["ls-tree", "-r", "--name-only", &s.commit]).output(), "t").unwrap();
assert_eq!(files.lines().collect::<Vec<_>>(), ["pkg/Cargo.toml", "pkg/src/lib.rs", "pkg/src/new.rs"]);
let parents = stdout(git_raw(Some(&staging)).args(["rev-list", "--parents", "-n1", &s.commit]).output(), "t").unwrap();
assert_eq!(parents.split_whitespace().count(), 1, "snapshot must be parentless");
assert_eq!(before, camp_git_state(p), "camp index/refs/status must be untouched");
}
#[test]
fn clean_tree_snapshot_is_not_dirty() {
let c = camp();
let staging = c.path().join("staging.git");
assert!(!snapshot(c.path(), &staging, SnapshotScope::Paths(&[PathBuf::from("pkg")]), ChainParent::NewestQedRef).unwrap().dirty);
}
#[test]
fn existing_commit_resolves_a_staged_commit_and_refuses_an_unknown_one() {
let c = camp();
let staging = c.path().join("staging.git");
let s = snapshot(c.path(), &staging, SnapshotScope::Paths(&[PathBuf::from("pkg")]), ChainParent::None).unwrap();
let e = existing_commit(&staging, &s.commit[..12]).unwrap();
assert_eq!((e.commit.as_str(), e.dirty), (s.commit.as_str(), false));
assert!(existing_commit(&staging, &"0".repeat(40)).is_err());
}
#[test]
fn snapshot_of_nothing_is_refused() {
let c = camp();
let staging = c.path().join("staging.git");
assert!(snapshot(c.path(), &staging, SnapshotScope::Paths(&[PathBuf::from("nope")]), ChainParent::NewestQedRef).is_err());
}
#[test]
fn context_url_round_trips_and_tar_urls_are_not_git() {
let r = run_ref("run 1", "build/x");
assert_eq!(r, "refs/qed/run-1/build-x");
let u = context_url("https://h:4330/yah.git", &r);
assert_eq!(parse_context_url(&u), Some(("https://h:4330/yah.git", r.as_str())));
assert_eq!(parse_context_url("https://cdn.yah.dev/qed-context/k.tar.gz"), None);
}
#[test]
fn total_objects_parses_git_progress() {
let s = "remote: Enumerating objects: 5, done.\rremote: Total 4 (delta 0), reused 0 (delta 0), pack-reused 0\n";
assert_eq!(parse_total_objects(s), Some(4));
assert_eq!(parse_total_objects("From x\n = [up to date] a -> b\n"), None);
}
/// Push → fetch → checkout over a local bare remote (the fleet-git e2e,
/// with real tokens, lives in the yah CLI's tests); the second snapshot
/// after a one-file edit moves only the new blob + its tree chain + commit.
#[test]
fn push_fetch_checkout_round_trip_moves_only_the_delta() {
let c = camp();
let p = c.path();
std::fs::write(p.join("pkg/src/new.rs"), "// untracked\n").unwrap();
let remote_dir = tempfile::tempdir().unwrap();
sh(remote_dir.path(), &["init", "--bare", "-q"]);
let remote = format!("file://{}", remote_dir.path().display());
let staging = p.join("staging.git");
let cache = tempfile::tempdir().unwrap();
let cache_repo = cache.path().join("yah.git");
let s1 = snapshot(p, &staging, SnapshotScope::Paths(&[PathBuf::from("pkg")]), ChainParent::NewestQedRef).unwrap();
push_snapshot(&staging, &remote, &s1.commit, &run_ref("r1", "s"), &[]).unwrap();
let ws1 = tempfile::tempdir().unwrap();
let f1 = fetch_checkout(&cache_repo, &context_url(&remote, &run_ref("r1", "s")), None, ws1.path()).unwrap();
assert_eq!(f1.oid, s1.commit);
assert_eq!(std::fs::read_to_string(ws1.path().join("pkg/src/new.rs")).unwrap(), "// untracked\n");
assert!(!ws1.path().join("other").exists());
std::fs::write(p.join("pkg/src/lib.rs"), "pub fn b() {}\n").unwrap();
let s2 = snapshot(p, &staging, SnapshotScope::Paths(&[PathBuf::from("pkg")]), ChainParent::NewestQedRef).unwrap();
push_snapshot(&staging, &remote, &s2.commit, &run_ref("r2", "s"), &[]).unwrap();
let ws2 = tempfile::tempdir().unwrap();
let f2 = fetch_checkout(&cache_repo, &context_url(&remote, &run_ref("r2", "s")), None, ws2.path()).unwrap();
assert_eq!(std::fs::read_to_string(ws2.path().join("pkg/src/lib.rs")).unwrap(), "pub fn b() {}\n");
// commit + root tree + pkg tree + pkg/src tree + one blob.
assert!(f2.objects.is_some_and(|n| n <= 5), "second fetch moved {:?} objects:\n{}", f2.objects, f2.log);
}
/// Whole-tree scope: untracked-not-ignored rides, tracked-but-ignored rides,
/// untracked-ignored stays out, and an excluded path is absent.
#[test]
fn whole_tree_snapshot_honours_ignore_and_exclusions() {
let c = camp();
let p = c.path();
std::fs::write(p.join(".gitignore"), "target/\n*.keep\n").unwrap();
std::fs::write(p.join("tracked.keep"), "t\n").unwrap();
sh(p, &["add", "-f", "tracked.keep", ".gitignore"]);
sh(p, &["commit", "-q", "-m", "ignored-but-tracked"]);
std::fs::write(p.join("tracked.keep"), "t2\n").unwrap();
std::fs::write(p.join("other/new"), "n\n").unwrap();
std::fs::write(p.join("untracked.keep"), "u\n").unwrap();
std::fs::create_dir_all(p.join("target")).unwrap();
std::fs::write(p.join("target/junk"), "j").unwrap();
let before = camp_git_state(p);
let st = tempfile::tempdir().unwrap();
let staging = st.path().join("staging.git");
let ex = [PathBuf::from("pkg/src")];
let s = snapshot(p, &staging, SnapshotScope::WholeTree { exclude: &ex }, ChainParent::None).unwrap();
let files = stdout(git_raw(Some(&staging)).args(["ls-tree", "-r", "--name-only", &s.commit]).output(), "t").unwrap();
let files: Vec<_> = files.lines().collect();
assert_eq!(files, [".gitignore", "other/new", "other/x", "pkg/Cargo.toml", "tracked.keep"]);
let blob = stdout(git_raw(Some(&staging)).args(["show", &format!("{}:tracked.keep", s.commit)]).output(), "t").unwrap();
assert_eq!(blob, "t2");
assert!(s.dirty);
assert_eq!(before, camp_git_state(p), "camp index/refs/status must be untouched");
}
/// Chained whole-tree snapshots: a one-file edit pushes one tree delta, not
/// the whole tree (W363 §6's 7-of-7 shape, counted as new objects).
#[test]
fn chained_whole_tree_snapshot_pushes_only_the_delta() {
let c = camp();
let p = c.path();
std::fs::write(p.join("pkg/src/new.rs"), "// untracked\n").unwrap();
let remote_dir = tempfile::tempdir().unwrap();
sh(remote_dir.path(), &["init", "--bare", "-q"]);
let remote = format!("file://{}", remote_dir.path().display());
let st = tempfile::tempdir().unwrap();
let staging = st.path().join("staging.git");
let count = |d: &Path| -> u64 {
stdout(git_raw(Some(d)).args(["count-objects", "-v"]).output(), "t")
.unwrap()
.lines()
.filter_map(|l| {
let (k, v) = l.split_once(": ")?;
matches!(k, "count" | "in-pack").then(|| v.trim().parse::<u64>().ok()).flatten()
})
.sum()
};
let mk = |ex: &[PathBuf]| {
snapshot(p, &staging, SnapshotScope::WholeTree { exclude: ex }, ChainParent::Named("refs/heads/camp")).unwrap()
};
let s1 = mk(&[]);
push_snapshot(&staging, &remote, &s1.commit, "refs/heads/camp", &[]).unwrap();
sh(&staging, &["update-ref", "refs/heads/camp", &s1.commit]);
let first = count(remote_dir.path());
assert!(first >= 6, "first push carries the whole tree ({first} objects)");
std::fs::write(p.join("pkg/src/lib.rs"), "pub fn b() {}\n").unwrap();
let s2 = mk(&[]);
let parents = stdout(git_raw(Some(&staging)).args(["rev-list", "--parents", "-n1", &s2.commit]).output(), "t").unwrap();
assert!(parents.contains(&s1.commit), "chained on refs/heads/camp");
push_snapshot(&staging, &remote, &s2.commit, "refs/heads/camp", &[]).unwrap();
let moved = count(remote_dir.path()) - first;
// commit + root tree + pkg tree + pkg/src tree + one blob.
assert!(moved <= 5, "one-file edit moved {moved} new objects (whole tree is {first})");
}
fn tree_of(staging: &Path, commit: &str) -> String {
stdout(git_raw(Some(staging)).args(["rev-parse", &format!("{commit}^{{tree}}")]).output(), "t").unwrap()
}
/// A kept index changes what a whole-tree snapshot costs, never what it
/// holds: after each change below, a snapshot through the kept index equals
/// one through a fresh index.
#[test]
fn a_kept_index_snapshots_the_same_tree_as_a_fresh_one() {
let c = camp();
let p = c.path();
std::fs::write(p.join(".gitignore"), "*.keep\n").unwrap();
std::fs::write(p.join("tracked.keep"), "t\n").unwrap();
sh(p, &["add", "-f", "tracked.keep", ".gitignore"]);
sh(p, &["commit", "-q", "-m", "ignored-but-tracked"]);
let st = tempfile::tempdir().unwrap();
let kept = st.path().join("kept.git");
let ex = [PathBuf::from("other/x")];
let same = |what: &str| {
let a = snapshot(p, &kept, SnapshotScope::WholeTree { exclude: &ex }, ChainParent::None).unwrap();
let fresh_dir = tempfile::tempdir().unwrap();
let fresh = fresh_dir.path().join("fresh.git");
let b = snapshot(p, &fresh, SnapshotScope::WholeTree { exclude: &ex }, ChainParent::None).unwrap();
assert_eq!(tree_of(&kept, &a.commit), tree_of(&fresh, &b.commit), "{what}");
assert_eq!(a.dirty, b.dirty, "{what}");
};
same("first snapshot");
assert!(kept.join("snapshot-index").read_dir().unwrap().count() >= 2, "the index and its lock are kept");
std::fs::write(p.join("pkg/src/lib.rs"), "pub fn b() {}\n").unwrap();
same("a same-size edit");
std::fs::remove_file(p.join("tracked.keep")).unwrap();
same("a tracked-but-ignored file deleted");
// `add -A` alone skips an ignored file the index no longer has; the
// seed from base is what brings it back.
std::fs::write(p.join("tracked.keep"), "t\n").unwrap();
same("the same file restored");
std::fs::write(p.join("other/new"), "n\n").unwrap();
same("an untracked file");
sh(p, &["add", "-A"]);
sh(p, &["commit", "-q", "-m", "base moves"]);
same("a moved base");
std::fs::write(p.join("pkg/src/lib.rs"), "pub fn c() {}\n").unwrap();
same("an edit on the moved base");
}
/// The kept index is a cache: held by another snapshot of the same tree,
/// unreadable, or beside a dead run's `index.lock`, the snapshot still
/// succeeds with the same tree.
#[test]
fn a_held_or_broken_kept_index_still_snapshots() {
let c = camp();
let p = c.path();
let st = tempfile::tempdir().unwrap();
let staging = st.path().join("staging.git");
let whole = || snapshot(p, &staging, SnapshotScope::WholeTree { exclude: &[] }, ChainParent::None).unwrap();
let want = tree_of(&staging, &whole().commit);
let held = KeptIndex::acquire(&staging, p).expect("free after the snapshot");
assert!(KeptIndex::acquire(&staging, p).is_none(), "one holder at a time");
assert_eq!(tree_of(&staging, &whole().commit), want, "held elsewhere: a temp index");
let path = held.path.clone();
drop(held);
std::fs::write(&path, b"not an index").unwrap();
std::fs::write(path.with_extension("lock"), b"").unwrap();
assert_eq!(tree_of(&staging, &whole().commit), want, "corrupt, beside a stale index.lock");
}
fn kept_entry_files(staging: &Path) -> Vec<String> {
let mut v: Vec<String> = std::fs::read_dir(staging.join("snapshot-index"))
.unwrap()
.flatten()
.map(|e| e.file_name().to_string_lossy().into_owned())
.collect();
v.sort();
v
}
/// Acquiring beside entries of retired worktrees sweeps them; a live
/// worktree's entry stays.
#[test]
fn a_retired_worktrees_kept_index_is_swept_and_a_live_one_kept() {
let a = camp();
let b = camp();
let staging = tempfile::tempdir().unwrap();
let (live, retired) = (KeptIndex::acquire(staging.path(), a.path()).unwrap(), KeptIndex::acquire(staging.path(), b.path()).unwrap());
let (live_path, retired_path) = (live.path.clone(), retired.path.clone());
drop((live, retired));
std::fs::write(&retired_path, b"idx").unwrap();
std::fs::write(&live_path, b"idx").unwrap();
let retired_name = retired_path.file_name().unwrap().to_string_lossy().into_owned();
drop(b);
let c = camp();
let _third = KeptIndex::acquire(staging.path(), c.path()).unwrap();
let files = kept_entry_files(staging.path());
assert!(files.iter().all(|f| !f.starts_with(&retired_name)), "{files:?}");
assert!(live_path.exists() && live_path.with_extension("flock").exists(), "{files:?}");
}
#[test]
fn a_held_entry_is_not_swept_and_never_waited_on() {
let a = camp();
let b = camp();
let staging = tempfile::tempdir().unwrap();
let held = KeptIndex::acquire(staging.path(), b.path()).unwrap();
let held_path = held.path.clone();
drop(a);
std::fs::remove_dir_all(b.path()).unwrap();
let c = camp();
let _third = KeptIndex::acquire(staging.path(), c.path()).unwrap();
assert!(held_path.with_extension("flock").exists(), "held flock: skipped");
assert!(held_path.with_extension("worktree").exists());
}
#[test]
fn a_sidecarless_entry_is_dropped_only_when_old() {
let staging = tempfile::tempdir().unwrap();
let dir = staging.path().join("snapshot-index");
std::fs::create_dir_all(&dir).unwrap();
for ext in ["", ".flock"] {
std::fs::write(dir.join(format!("deadbeefdeadbeef{ext}")), b"x").unwrap();
}
sweep_retired(&dir, "own", std::time::Duration::from_secs(3600));
assert_eq!(std::fs::read_dir(&dir).unwrap().count(), 2, "young legacy entry kept");
std::thread::sleep(std::time::Duration::from_millis(50));
sweep_retired(&dir, "own", std::time::Duration::from_millis(10));
assert_eq!(std::fs::read_dir(&dir).unwrap().count(), 0, "old legacy entry dropped");
}
/// The same script materializes a tar source context (`file://` stands in
/// for the CDN URL; curl treats them alike).
#[test]
fn the_fetch_script_extracts_a_tar_context() {
let src = tempfile::tempdir().unwrap();
std::fs::create_dir_all(src.path().join("pkg")).unwrap();
std::fs::write(src.path().join("pkg/a.txt"), "tar\n").unwrap();
let out = tempfile::tempdir().unwrap();
let tgz = out.path().join("ctx.tar.gz");
let st = Command::new("tar").arg("-czf").arg(&tgz).arg("-C").arg(src.path()).arg("pkg").status().unwrap();
assert!(st.success());
let ws = tempfile::tempdir().unwrap();
let cache = tempfile::tempdir().unwrap();
fetch_checkout(&cache.path().join("c.git"), &format!("file://{}", tgz.display()), None, ws.path()).unwrap();
assert_eq!(std::fs::read_to_string(ws.path().join("pkg/a.txt")).unwrap(), "tar\n");
assert!(!cache.path().join("c.git").exists(), "the tar path must not touch the git cache");
}
#[test]
fn failed_fetch_names_the_tar_override() {
let cache = tempfile::tempdir().unwrap();
let ws = tempfile::tempdir().unwrap();
let e = fetch_checkout(&cache.path().join("c.git"), "git+http://127.0.0.1:9/x.git#refs/qed/a/b", None, ws.path())
.unwrap_err()
.to_string();
assert!(e.contains("YAH_QED_SOURCE_TRANSPORT=tar"), "{e}");
}
/// R969-F7 return path over a local bare remote (the fleet-git e2e with
/// real tokens lives in the yah CLI): a worker edit + a new file ride
/// `yah-source-ship` back, harvest lands them as uncommitted changes, and
/// the camp's index and refs are untouched.
#[test]
fn ship_then_harvest_applies_the_return_diff_uncommitted() {
let c = camp();
let p = c.path();
let remote_dir = tempfile::tempdir().unwrap();
sh(remote_dir.path(), &["init", "--bare", "-q"]);
let remote = format!("file://{}", remote_dir.path().display());
let st = tempfile::tempdir().unwrap();
let staging = st.path().join("yah.git");
let cache = tempfile::tempdir().unwrap();
let cache_repo = cache.path().join("src.git");
let s = snapshot(p, &staging, SnapshotScope::Paths(&[PathBuf::from("pkg")]), ChainParent::NewestQedRef).unwrap();
let snap_ref = run_ref("r1", "build");
push_snapshot(&staging, &remote, &s.commit, &snap_ref, &[]).unwrap();
let url = context_url(&remote, &snap_ref);
let ws = tempfile::tempdir().unwrap();
fetch_checkout(&cache_repo, &url, None, ws.path()).unwrap();
std::fs::write(ws.path().join("pkg/src/lib.rs"), "pub fn fixed() {}\n").unwrap();
std::fs::write(ws.path().join("pkg/src/added.rs"), "// new on the worker\n").unwrap();
let ret = return_ref("r1", "build");
assert_eq!(ret, "refs/qed/r1/build.return");
let pushed = ship_return(&cache_repo, &url, &s.commit, &ret, "unused-on-file-remote", ws.path()).unwrap();
assert_eq!(pushed.len(), 40, "{pushed}");
let before = camp_git_state(p);
let h = harvest_return(p, &staging, &remote, &ret, &s.commit, &[]).unwrap();
assert_eq!(h.commit, pushed);
assert_eq!(h.files, ["pkg/src/added.rs", "pkg/src/lib.rs"]);
assert_eq!(std::fs::read_to_string(p.join("pkg/src/lib.rs")).unwrap(), "pub fn fixed() {}\n");
assert_eq!(std::fs::read_to_string(p.join("pkg/src/added.rs")).unwrap(), "// new on the worker\n");
let after = camp_git_state(p);
assert_eq!((&before.1, &before.2), (&after.1, &after.2), "camp refs and index must be untouched");
assert!(after.0.contains(" M pkg/src/lib.rs") && after.0.contains("?? pkg/src/added.rs"), "{}", after.0);
}
/// The camp moved under the same line since the snapshot: harvest refuses
/// and writes nothing.
#[test]
fn harvest_refuses_on_conflict_and_writes_nothing() {
let c = camp();
let p = c.path();
let remote_dir = tempfile::tempdir().unwrap();
sh(remote_dir.path(), &["init", "--bare", "-q"]);
let remote = format!("file://{}", remote_dir.path().display());
let st = tempfile::tempdir().unwrap();
let staging = st.path().join("yah.git");
let cache = tempfile::tempdir().unwrap();
let cache_repo = cache.path().join("src.git");
let s = snapshot(p, &staging, SnapshotScope::Paths(&[PathBuf::from("pkg")]), ChainParent::NewestQedRef).unwrap();
let snap_ref = run_ref("r1", "build");
push_snapshot(&staging, &remote, &s.commit, &snap_ref, &[]).unwrap();
let url = context_url(&remote, &snap_ref);
let ws = tempfile::tempdir().unwrap();
fetch_checkout(&cache_repo, &url, None, ws.path()).unwrap();
std::fs::write(ws.path().join("pkg/src/lib.rs"), "pub fn worker() {}\n").unwrap();
std::fs::write(ws.path().join("pkg/Cargo.toml"), "[package]\nname = \"w\"\n").unwrap();
let ret = return_ref("r1", "build");
ship_return(&cache_repo, &url, &s.commit, &ret, "t", ws.path()).unwrap();
std::fs::write(p.join("pkg/src/lib.rs"), "pub fn camp() {}\n").unwrap();
let e = harvest_return(p, &staging, &remote, &ret, &s.commit, &[]).unwrap_err().to_string();
assert!(e.contains("does not apply"), "{e}");
assert_eq!(std::fs::read_to_string(p.join("pkg/src/lib.rs")).unwrap(), "pub fn camp() {}\n");
assert_eq!(std::fs::read_to_string(p.join("pkg/Cargo.toml")).unwrap(), "[package]\n", "a refused harvest is all-or-nothing");
}
#[test]
fn harvest_refuses_a_return_not_built_on_the_snapshot() {
let c = camp();
let p = c.path();
let remote_dir = tempfile::tempdir().unwrap();
sh(remote_dir.path(), &["init", "--bare", "-q"]);
let remote = format!("file://{}", remote_dir.path().display());
let staging = p.join("staging.git");
let s1 = snapshot(p, &staging, SnapshotScope::Paths(&[PathBuf::from("pkg")]), ChainParent::None).unwrap();
std::fs::write(p.join("other/x"), "y\n").unwrap();
let s2 = snapshot(p, &staging, SnapshotScope::Paths(&[PathBuf::from("other")]), ChainParent::None).unwrap();
push_snapshot(&staging, &remote, &s2.commit, "refs/qed/r/s.return", &[]).unwrap();
let e = harvest_return(p, &staging, &remote, "refs/qed/r/s.return", &s1.commit, &[]).unwrap_err().to_string();
assert!(e.contains("does not descend"), "{e}");
}
}