use std::collections::BTreeSet;
use std::ffi::OsStr;
use std::fs::{self, File, OpenOptions};
use std::io;
use std::path::{Path, PathBuf};
use std::process::Command;
use std::time::{SystemTime, UNIX_EPOCH};
use flate2::Compression;
use flate2::write::GzEncoder;
use ignore::{DirEntry, WalkBuilder};
use tar::{Builder, Header};
use crate::shim::config;
const ALWAYS_IGNORED_PARTS: &[&str] = &[
".falsegreen",
".git",
".hypothesis",
".mypy_cache",
".pytest_cache",
".ruff_cache",
".uv-cache",
".venv",
"__pycache__",
];
const ROOT_IGNORED_PARTS: &[&str] = &["build", "dist", "target"];
const MAX_SUBMODULE_DEPTH: usize = 8;
pub fn sync(workspace: &Path, key: &str) -> Result<(), String> {
let canonical = workspace
.canonicalize()
.map_err(|error| format!("cannot resolve workspace {}: {error}", workspace.display()))?;
if !canonical.is_dir() {
return Err(format!(
"workspace is not a directory: {}",
canonical.display()
));
}
let archive =
TemporaryArchive::new().map_err(|error| format!("cannot create snapshot: {error}"))?;
create_archive(&canonical, archive.path())
.map_err(|error| format!("cannot snapshot workspace: {error}"))?;
let output = Command::new("curl")
.arg("-sS")
.arg("-f")
.arg("-X")
.arg("PUT")
.arg(config::workspace_url())
.arg("-H")
.arg("Content-Type: application/gzip")
.arg("-H")
.arg(format!("X-API-KEY: {key}"))
.arg("--data-binary")
.arg(format!("@{}", archive.path().display()))
.arg("--max-time")
.arg("120")
.output()
.map_err(|error| format!("failed to upload workspace: {error}"))?;
if !output.status.success() {
let detail = String::from_utf8_lossy(&output.stderr).trim().to_string();
return Err(if detail.is_empty() {
"workspace upload failed".to_string()
} else {
format!("workspace upload failed: {detail}")
});
}
Ok(())
}
fn create_archive(workspace: &Path, destination: &Path) -> io::Result<()> {
let file = OpenOptions::new()
.write(true)
.truncate(true)
.open(destination)?;
let encoder = GzEncoder::new(file, Compression::default());
let mut archive = Builder::new(encoder);
archive.follow_symlinks(false);
append_workspace(&mut archive, workspace)?;
let encoder = archive.into_inner()?;
encoder.finish()?;
Ok(())
}
fn git_output(workspace: &Path, arguments: &[&str]) -> io::Result<Option<Vec<u8>>> {
let output = Command::new("git")
.args(arguments)
.current_dir(workspace)
.output()?;
Ok(output.status.success().then_some(output.stdout))
}
fn append_bytes<W: io::Write>(
archive: &mut Builder<W>,
path: &Path,
content: &[u8],
) -> io::Result<()> {
let mut header = Header::new_gnu();
header.set_size(content.len() as u64);
header.set_mode(0o600);
header.set_mtime(0);
header.set_cksum();
archive.append_data(&mut header, path, content)
}
fn append_git_identity<W: io::Write>(
archive: &mut Builder<W>,
workspace: &Path,
archive_prefix: &Path,
) -> io::Result<()> {
let Some(mut commit) = git_output(workspace, &["rev-parse", "--verify", "HEAD"])? else {
return Ok(());
};
commit.truncate(
commit
.iter()
.position(|byte| *byte == b'\n')
.unwrap_or(commit.len()),
);
if !matches!(commit.len(), 40 | 64) || !commit.iter().all(u8::is_ascii_hexdigit) {
return Ok(());
}
commit.make_ascii_lowercase();
let Some(git_dir_raw) = git_output(workspace, &["rev-parse", "--absolute-git-dir"])? else {
return Ok(());
};
let git_dir_text = String::from_utf8_lossy(&git_dir_raw);
let git_dir = PathBuf::from(git_dir_text.trim());
let index = match fs::read(git_dir.join("index")) {
Ok(index) if index.len() <= 64 * 1024 * 1024 => index,
_ => return Ok(()),
};
let reference = git_output(workspace, &["symbolic-ref", "-q", "HEAD"])?
.and_then(|value| String::from_utf8(value).ok())
.map(|value| value.trim().to_string())
.filter(|value| {
value.starts_with("refs/heads/")
&& !value.contains("..")
&& value
.bytes()
.all(|byte| byte.is_ascii_alphanumeric() || b"/._-".contains(&byte))
});
let repository = git_output(workspace, &["remote", "get-url", "origin"])?
.and_then(|value| String::from_utf8(value).ok())
.and_then(|value| canonical_repository_url(value.trim()));
let config = repository
.map(|url| format!("[remote \"origin\"]\n\turl = {url}\n"))
.unwrap_or_default();
let metadata_root = archive_prefix.join(".git");
append_bytes(archive, &metadata_root.join("index"), &index)?;
append_bytes(archive, &metadata_root.join("config"), config.as_bytes())?;
if let Some(reference) = reference {
append_bytes(
archive,
&metadata_root.join("HEAD"),
format!("ref: {reference}\n").as_bytes(),
)?;
append_bytes(
archive,
&metadata_root.join(reference),
&[commit.as_slice(), b"\n"].concat(),
)?;
} else {
append_bytes(
archive,
&metadata_root.join("HEAD"),
&[commit.as_slice(), b"\n"].concat(),
)?;
}
Ok(())
}
fn canonical_repository_url(value: &str) -> Option<String> {
if value.is_empty()
|| value.len() > 4096
|| value
.chars()
.any(|character| character.is_control() || character.is_whitespace())
{
return None;
}
let (host, path) = if let Some(rest) = value
.strip_prefix("https://")
.or_else(|| value.strip_prefix("http://"))
.or_else(|| value.strip_prefix("ssh://"))
{
let (authority, path) = rest.split_once('/')?;
(
authority
.rsplit_once('@')
.map_or(authority, |(_, host)| host),
path,
)
} else {
let (_, location) = value.rsplit_once('@')?;
location.split_once(':')?
};
if host.is_empty()
|| path.is_empty()
|| !host
.bytes()
.all(|byte| byte.is_ascii_alphanumeric() || b".-".contains(&byte))
{
return None;
}
Some(format!("https://{host}/{}", path.trim_start_matches('/')))
}
fn ignored_relative(path: &Path) -> bool {
let mut components = path.components();
let Some(first) = components.next().and_then(|part| part.as_os_str().to_str()) else {
return true;
};
ROOT_IGNORED_PARTS.contains(&first)
|| std::iter::once(first)
.chain(components.filter_map(|part| part.as_os_str().to_str()))
.any(|name| name.starts_with(".bench-") || ALWAYS_IGNORED_PARTS.contains(&name))
}
fn git_snapshot_paths(workspace: &Path) -> io::Result<Option<Vec<PathBuf>>> {
let Some(output) = git_output(
workspace,
&[
"ls-files",
"-z",
"--cached",
"--others",
"--exclude-standard",
],
)?
else {
return Ok(None);
};
let mut paths = BTreeSet::new();
for encoded in output
.split(|byte| *byte == 0)
.filter(|value| !value.is_empty())
{
let relative = PathBuf::from(
String::from_utf8(encoded.to_vec())
.map_err(|_| io::Error::other("Git path is not valid UTF-8"))?,
);
if relative.is_absolute()
|| relative
.components()
.any(|part| !matches!(part, std::path::Component::Normal(_)))
{
return Err(io::Error::other("Git returned an unsafe workspace path"));
}
if !ignored_relative(&relative) {
paths.insert(relative);
}
}
Ok(Some(paths.into_iter().collect()))
}
fn git_commit(workspace: &Path, arguments: &[&str]) -> io::Result<Option<String>> {
Ok(git_output(workspace, arguments)?
.and_then(|value| String::from_utf8(value).ok())
.map(|value| value.trim().to_ascii_lowercase())
.filter(|value| {
matches!(value.len(), 40 | 64) && value.bytes().all(|byte| byte.is_ascii_hexdigit())
}))
}
fn append_git_workspace<W: io::Write>(
archive: &mut Builder<W>,
workspace: &Path,
archive_prefix: &Path,
depth: usize,
) -> io::Result<()> {
if depth > MAX_SUBMODULE_DEPTH {
return Err(io::Error::other(
"Git submodule nesting exceeds the supported depth",
));
}
let paths = git_snapshot_paths(workspace)?
.ok_or_else(|| io::Error::other("initialized Git submodule metadata is unavailable"))?;
for relative in paths {
let path = workspace.join(&relative);
let archive_path = archive_prefix.join(&relative);
let metadata = match fs::symlink_metadata(&path) {
Ok(metadata) => metadata,
Err(error) if error.kind() == io::ErrorKind::NotFound => continue,
Err(error) => return Err(error),
};
let file_type = metadata.file_type();
if file_type.is_file() || file_type.is_symlink() {
archive.append_path_with_name(path, archive_path)?;
} else if file_type.is_dir() {
let index_expression = format!(":{}", relative.to_string_lossy());
let expected = git_commit(workspace, &["rev-parse", "--verify", &index_expression])?
.ok_or_else(|| {
io::Error::other(format!(
"workspace directory is not a Git submodule: {}",
relative.display()
))
})?;
let observed =
git_commit(&path, &["rev-parse", "--verify", "HEAD"])?.ok_or_else(|| {
io::Error::other(format!(
"Git submodule is not initialized: {}",
relative.display()
))
})?;
if observed != expected {
return Err(io::Error::other(format!(
"Git submodule {} is checked out at {observed}, expected {expected}",
relative.display()
)));
}
append_git_workspace(archive, &path, &archive_path, depth + 1)?;
} else {
return Err(io::Error::other(format!(
"unsupported Git workspace entry: {}",
relative.display()
)));
}
}
append_git_identity(archive, workspace, archive_prefix)
}
fn append_workspace<W: io::Write>(archive: &mut Builder<W>, workspace: &Path) -> io::Result<()> {
if git_snapshot_paths(workspace)?.is_some() {
return append_git_workspace(archive, workspace, Path::new(""), 0);
}
let mut walker = WalkBuilder::new(workspace);
walker
.hidden(false)
.follow_links(false)
.git_ignore(true)
.git_global(false)
.git_exclude(true)
.parents(true)
.sort_by_file_path(|left, right| left.cmp(right))
.filter_entry(|entry| entry.depth() == 0 || !ignored(entry));
for entry in walker.build() {
let entry = entry.map_err(io::Error::other)?;
if entry.depth() == 0 {
continue;
}
let path = entry.path();
let relative = path.strip_prefix(workspace).map_err(io::Error::other)?;
let metadata = fs::symlink_metadata(path)?;
let file_type = metadata.file_type();
if file_type.is_dir() {
archive.append_dir(relative, path)?;
} else if file_type.is_file() || file_type.is_symlink() {
archive.append_path_with_name(path, relative)?;
} else {
return Err(io::Error::other(format!(
"unsupported workspace entry: {}",
relative.display()
)));
}
}
Ok(())
}
fn ignored(entry: &DirEntry) -> bool {
let name = entry.file_name();
name.to_str()
.is_some_and(|name| name.starts_with(".bench-"))
|| ALWAYS_IGNORED_PARTS
.iter()
.any(|ignored| name == OsStr::new(ignored))
|| (entry.depth() == 1
&& ROOT_IGNORED_PARTS
.iter()
.any(|ignored| name == OsStr::new(ignored)))
}
struct TemporaryArchive {
path: PathBuf,
}
impl TemporaryArchive {
fn new() -> io::Result<Self> {
let nonce = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_nanos();
let path = std::env::temp_dir().join(format!(
"falsegreen-workspace-{}-{nonce}.tar.gz",
std::process::id()
));
File::create_new(&path)?;
Ok(Self { path })
}
fn path(&self) -> &Path {
&self.path
}
}
impl Drop for TemporaryArchive {
fn drop(&mut self) {
let _ = fs::remove_file(&self.path);
}
}
#[cfg(test)]
mod tests {
use super::*;
use flate2::read::GzDecoder;
use std::io::Read;
fn test_directory() -> PathBuf {
let nonce = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap()
.as_nanos();
let directory = std::env::temp_dir().join(format!(
"falsegreen-workspace-test-{}-{nonce}",
std::process::id()
));
fs::create_dir(&directory).unwrap();
directory
}
fn run_git(workspace: &Path, arguments: &[&str]) {
assert!(
Command::new("git")
.args(arguments)
.current_dir(workspace)
.status()
.unwrap()
.success(),
"git command failed: {arguments:?}"
);
}
fn commit_all(workspace: &Path, message: &str) {
run_git(workspace, &["add", "-A"]);
run_git(
workspace,
&[
"-c",
"user.name=FalseGreen Test",
"-c",
"user.email=test@falsegreen.invalid",
"commit",
"-q",
"-m",
message,
],
);
}
fn add_submodule(workspace: &Path, origin: &Path, destination: &str) {
run_git(
workspace,
&[
"-c",
"protocol.file.allow=always",
"submodule",
"add",
"-q",
origin.to_str().unwrap(),
destination,
],
);
}
fn submodule_fixture() -> (PathBuf, PathBuf) {
let root = test_directory();
let child = root.join("child-origin");
let parent = root.join("parent");
fs::create_dir(&child).unwrap();
fs::create_dir(&parent).unwrap();
run_git(&child, &["init", "-q"]);
fs::write(child.join("lib.rs"), "pub fn child() {}\n").unwrap();
commit_all(&child, "child fixture");
run_git(&parent, &["init", "-q"]);
fs::write(parent.join("main.rs"), "fn main() {}\n").unwrap();
commit_all(&parent, "parent fixture");
add_submodule(&parent, &child, "vendor/child");
run_git(
&parent,
&[
"-c",
"protocol.file.allow=always",
"submodule",
"update",
"--init",
"--recursive",
],
);
commit_all(&parent, "add child submodule");
(root, parent)
}
#[test]
fn archive_contains_source_and_excludes_generated_state() {
let workspace = test_directory();
fs::write(workspace.join("main.rs"), "fn main() {}\n").unwrap();
fs::write(workspace.join("draft.txt"), "untracked source\n").unwrap();
fs::write(workspace.join(".gitignore"), ".bench-*/\n").unwrap();
fs::create_dir(workspace.join("target")).unwrap();
fs::write(workspace.join("target/output"), "ignored").unwrap();
fs::create_dir(workspace.join(".bench-generated")).unwrap();
fs::write(workspace.join(".bench-generated/output"), "ignored").unwrap();
fs::create_dir(workspace.join("src")).unwrap();
fs::write(workspace.join("src/lib.rs"), "pub fn works() {}\n").unwrap();
fs::create_dir_all(workspace.join("src/pip/_internal/operations/build")).unwrap();
fs::write(
workspace.join("src/pip/_internal/operations/build/__init__.py"),
"# tracked source package\n",
)
.unwrap();
let archive_path = workspace.parent().unwrap().join(format!(
"falsegreen-workspace-test-{}.tar.gz",
std::process::id()
));
File::create(&archive_path).unwrap();
create_archive(&workspace, &archive_path).unwrap();
let decoder = GzDecoder::new(File::open(&archive_path).unwrap());
let mut archive = tar::Archive::new(decoder);
let mut paths = Vec::new();
for entry in archive.entries().unwrap() {
let mut entry = entry.unwrap();
paths.push(entry.path().unwrap().to_string_lossy().to_string());
let mut sink = Vec::new();
entry.read_to_end(&mut sink).unwrap();
}
assert!(paths.contains(&"main.rs".to_string()));
assert!(paths.contains(&"draft.txt".to_string()));
assert!(paths.contains(&"src/lib.rs".to_string()));
assert!(paths.contains(&"src/pip/_internal/operations/build/__init__.py".to_string()));
assert!(!paths.iter().any(|path| path.starts_with("target")));
assert!(
!paths
.iter()
.any(|path| path.starts_with(".bench-generated"))
);
fs::remove_file(archive_path).unwrap();
fs::remove_dir_all(workspace).unwrap();
}
#[test]
fn archive_carries_minimal_git_identity_without_git_objects() {
let workspace = test_directory();
fs::write(workspace.join("main.rs"), "fn main() {}\n").unwrap();
fs::write(workspace.join(".gitignore"), "*.dat\n").unwrap();
fs::write(workspace.join("tracked.dat"), "tracked\n").unwrap();
fs::write(workspace.join("ignored.dat"), "ignored\n").unwrap();
let git = |arguments: &[&str]| {
Command::new("git")
.args(arguments)
.current_dir(&workspace)
.status()
.unwrap()
};
assert!(git(&["init", "-q"]).success());
assert!(git(&["add", "main.rs", ".gitignore"]).success());
assert!(git(&["add", "-f", "tracked.dat"]).success());
assert!(
git(&[
"remote",
"add",
"origin",
"git@github.com:falsegreen/fixture.git",
])
.success()
);
assert!(
git(&[
"-c",
"user.name=FalseGreen Test",
"-c",
"user.email=test@falsegreen.invalid",
"commit",
"-q",
"-m",
"fixture",
])
.success()
);
let archive_path = workspace.parent().unwrap().join(format!(
"falsegreen-workspace-git-test-{}.tar.gz",
std::process::id()
));
File::create(&archive_path).unwrap();
create_archive(&workspace, &archive_path).unwrap();
let decoder = GzDecoder::new(File::open(&archive_path).unwrap());
let mut archive = tar::Archive::new(decoder);
let paths: Vec<String> = archive
.entries()
.unwrap()
.map(|entry| entry.unwrap().path().unwrap().to_string_lossy().to_string())
.collect();
assert!(paths.contains(&".git/HEAD".to_string()));
assert!(paths.contains(&".git/index".to_string()));
assert!(paths.contains(&"tracked.dat".to_string()));
assert!(!paths.contains(&"ignored.dat".to_string()));
assert!(
paths
.iter()
.any(|path| path.starts_with(".git/refs/heads/"))
);
assert!(!paths.iter().any(|path| path.starts_with(".git/objects/")));
fs::remove_file(archive_path).unwrap();
fs::remove_dir_all(workspace).unwrap();
}
#[test]
fn archive_recursively_snapshots_pinned_git_submodules() {
let (root, workspace) = submodule_fixture();
fs::write(
workspace.join("vendor/child/untracked.txt"),
"source-bound local change\n",
)
.unwrap();
let archive_path = root.join("workspace.tar.gz");
File::create(&archive_path).unwrap();
create_archive(&workspace, &archive_path).unwrap();
let decoder = GzDecoder::new(File::open(&archive_path).unwrap());
let mut archive = tar::Archive::new(decoder);
let paths: Vec<String> = archive
.entries()
.unwrap()
.map(|entry| entry.unwrap().path().unwrap().to_string_lossy().to_string())
.collect();
assert!(paths.contains(&"vendor/child/lib.rs".to_string()));
assert!(paths.contains(&"vendor/child/untracked.txt".to_string()));
assert!(paths.contains(&"vendor/child/.git/HEAD".to_string()));
assert!(paths.contains(&"vendor/child/.git/index".to_string()));
assert!(paths.contains(&"vendor/child/.git/config".to_string()));
assert!(
!paths
.iter()
.any(|path| path.contains("/.git/objects/") || path.starts_with(".git/modules/"))
);
fs::remove_dir_all(root).unwrap();
}
#[test]
fn archive_rejects_submodule_checkout_that_does_not_match_gitlink() {
let (root, workspace) = submodule_fixture();
let child = workspace.join("vendor/child");
fs::write(child.join("lib.rs"), "pub fn changed() {}\n").unwrap();
commit_all(&child, "move checkout past parent gitlink");
let archive_path = root.join("workspace.tar.gz");
File::create(&archive_path).unwrap();
let error = create_archive(&workspace, &archive_path).unwrap_err();
assert!(error.to_string().contains("is checked out at"));
assert!(error.to_string().contains("expected"));
fs::remove_dir_all(root).unwrap();
}
#[test]
fn archive_recursively_snapshots_nested_git_submodules() {
let root = test_directory();
let leaf = root.join("leaf-origin");
let child = root.join("child-origin");
let parent = root.join("parent");
for repository in [&leaf, &child, &parent] {
fs::create_dir(repository).unwrap();
run_git(repository, &["init", "-q"]);
}
fs::write(leaf.join("leaf.rs"), "pub fn leaf() {}\n").unwrap();
commit_all(&leaf, "leaf fixture");
fs::write(child.join("child.rs"), "pub fn child() {}\n").unwrap();
commit_all(&child, "child fixture");
add_submodule(&child, &leaf, "deps/leaf");
commit_all(&child, "add leaf submodule");
fs::write(parent.join("main.rs"), "fn main() {}\n").unwrap();
commit_all(&parent, "parent fixture");
add_submodule(&parent, &child, "vendor/child");
run_git(
&parent,
&[
"-c",
"protocol.file.allow=always",
"submodule",
"update",
"--init",
"--recursive",
],
);
commit_all(&parent, "add child submodule");
let archive_path = root.join("workspace.tar.gz");
File::create(&archive_path).unwrap();
create_archive(&parent, &archive_path).unwrap();
let decoder = GzDecoder::new(File::open(&archive_path).unwrap());
let mut archive = tar::Archive::new(decoder);
let paths: Vec<String> = archive
.entries()
.unwrap()
.map(|entry| entry.unwrap().path().unwrap().to_string_lossy().to_string())
.collect();
assert!(paths.contains(&"vendor/child/deps/leaf/leaf.rs".to_string()));
assert!(paths.contains(&"vendor/child/deps/leaf/.git/index".to_string()));
fs::remove_dir_all(root).unwrap();
}
#[test]
fn repository_urls_are_canonicalized_without_credentials() {
assert_eq!(
canonical_repository_url("git@github.com:org/project.git").as_deref(),
Some("https://github.com/org/project.git")
);
assert_eq!(
canonical_repository_url("ssh://git@github.com/org/project.git").as_deref(),
Some("https://github.com/org/project.git")
);
assert_eq!(
canonical_repository_url("https://token@github.com/org/project.git").as_deref(),
Some("https://github.com/org/project.git")
);
assert!(canonical_repository_url("https://github.com/x\nsecret").is_none());
}
#[cfg(unix)]
#[test]
fn archive_preserves_symlinks_instead_of_reading_through_them() {
use std::os::unix::fs::symlink;
let workspace = test_directory();
fs::write(workspace.join("source.txt"), "source\n").unwrap();
symlink("source.txt", workspace.join("current")).unwrap();
let archive_path = workspace.parent().unwrap().join(format!(
"falsegreen-workspace-symlink-test-{}.tar.gz",
std::process::id()
));
File::create(&archive_path).unwrap();
create_archive(&workspace, &archive_path).unwrap();
let decoder = GzDecoder::new(File::open(&archive_path).unwrap());
let mut archive = tar::Archive::new(decoder);
let current = archive
.entries()
.unwrap()
.map(Result::unwrap)
.find(|entry| entry.path().unwrap() == Path::new("current"))
.unwrap();
assert!(current.header().entry_type().is_symlink());
assert_eq!(
current.link_name().unwrap().unwrap(),
Path::new("source.txt")
);
fs::remove_file(archive_path).unwrap();
fs::remove_dir_all(workspace).unwrap();
}
}