use std::collections::{HashSet, VecDeque};
use std::path::PathBuf;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Arc, Mutex, MutexGuard, OnceLock, PoisonError};
use crate::diff::FileDiff;
use crate::git::blobs::BlobReader;
use crate::git::credentials::Credentials;
use crate::git::files::{ChangedFile, ZERO_OID, is_lockfile};
use crate::git::repo::{PrRefs, Repo};
use crate::git::{GitError, Oid};
use futures::{StreamExt, TryStreamExt};
use tokio::sync::{mpsc, watch};
use tokio::task::JoinSet;
use crate::diff_screen::DiffOf;
use crate::state::{DiffMsg, Msg};
use crate::work::Failure;
#[derive(Debug, Clone)]
pub struct GitContext {
pub cache_root: PathBuf,
pub credentials: Credentials,
pub cwd: PathBuf,
}
#[derive(Debug)]
pub struct DiffFiles {
pub refs: PrRefs,
pub files: Vec<ChangedFile>,
pub generated: HashSet<String>,
}
#[derive(Debug)]
pub enum JobMsg {
Progress(String),
Files(Box<DiffFiles>),
File(usize, Arc<FileDiff>),
Failed(Failure),
Moves(Vec<crate::diff::moves::Move>),
Mapped(Vec<(crate::api::model::NodeId, Option<u32>)>),
Since(String, Result<HashSet<String>, Failure>),
}
pub type JobId = u64;
pub fn next_job() -> JobId {
static NEXT: AtomicU64 = AtomicU64::new(1);
NEXT.fetch_add(1, Ordering::Relaxed)
}
#[derive(Clone)]
pub struct JobTx {
pub tx: mpsc::UnboundedSender<Msg>,
pub of: DiffOf,
pub job: JobId,
}
impl JobTx {
pub fn send(&self, msg: JobMsg) -> bool {
self.tx
.send(Msg::Diff(self.of.clone(), DiffMsg::Job(self.job, msg)))
.is_ok()
}
}
pub type RepoGit = (Arc<Repo>, Arc<BlobReader>);
#[derive(Default)]
pub struct JobControl {
queue: Mutex<VecDeque<usize>>,
git: OnceLock<RepoGit>,
}
impl JobControl {
fn queue(&self) -> MutexGuard<'_, VecDeque<usize>> {
self.queue.lock().unwrap_or_else(PoisonError::into_inner)
}
pub fn prioritize(&self, files: &[usize]) {
let mut queue = self.queue();
for file in files.iter().rev() {
if let Some(pos) = queue.iter().position(|f| f == file) {
queue.remove(pos);
queue.push_front(*file);
}
}
}
fn pop(&self) -> Option<usize> {
self.queue().pop_front()
}
fn fill(&self, files: impl IntoIterator<Item = usize>) {
let mut queue = self.queue();
queue.clear();
queue.extend(files);
}
pub fn git(&self) -> Option<RepoGit> {
self.git.get().cloned()
}
fn front(&self) -> Option<usize> {
self.queue().front().copied()
}
}
pub type CommitRange = Option<(String, String)>;
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct PrBase {
pub branch: String,
pub oid: Option<Oid>,
}
pub async fn run(
ctx: GitContext,
base: PrBase,
range: CommitRange,
out: JobTx,
control: Arc<JobControl>,
) {
if let Err(err) = run_inner(&ctx, &base, range, &out, &control).await {
tracing::warn!(diff = %out.of, %err, "diff job failed");
out.send(JobMsg::Failed(err.into()));
}
}
async fn run_inner(
ctx: &GitContext,
base: &PrBase,
range: CommitRange,
out: &JobTx,
control: &Arc<JobControl>,
) -> Result<(), GitError> {
let of = &out.of;
let progress = {
let out = out.clone();
move |line: String| {
out.send(JobMsg::Progress(line));
}
};
let (owner, name) = (&of.repo().owner, &of.repo().name);
let repo = match Repo::find_local(&ctx.cwd, owner, name).await? {
Some(repo) => {
tracing::info!(path = %repo.path.display(), remote = repo.remote, "using local clone");
repo
}
None => {
Repo::open_cache(
&ctx.cache_root,
owner,
name,
&crate::host::current().git_url(owner, name),
ctx.credentials.clone(),
&progress,
)
.await?
}
};
let mut refs = match of {
DiffOf::Pr(pr) => {
repo.fetch_pr(pr.number, &base.branch, base.oid.as_ref(), &progress)
.await?
}
DiffOf::Commit(_, oid) => repo.commit_refs(oid, &progress).await?,
DiffOf::Range(_, from, to) => repo.range_refs(from, to, &progress).await?,
};
if let Some((from, to)) = range {
if let Ok(oid) = Oid::parse(&to) {
repo.ensure(&oid, &progress).await?;
}
refs.head = repo.rev_parse(&to).await?;
refs.merge_base = if from.is_empty() {
repo.merge_base(&refs.base, &refs.head).await?
} else {
repo.rev_parse(&from).await?
};
}
if let DiffOf::Pr(pr) = of
&& let Err(err) = repo.pin_seen(pr.number, &refs.head).await
{
tracing::warn!(%err, "could not pin head");
}
progress("Listing changed files".into());
let files = repo.changed_files(&refs.merge_base, &refs.head).await?;
let paths: Vec<&str> = files.iter().map(ChangedFile::path).collect();
let generated = repo
.generated_paths(&refs.head, &paths)
.await
.unwrap_or_else(|err| {
tracing::warn!(%err, "could not read linguist attributes");
HashSet::new()
});
let collapsed = |f: &ChangedFile| generated.contains(f.path()) || is_lockfile(f.path());
let (expanded, collapsed): (Vec<_>, Vec<_>) = (0..files.len())
.zip(&files)
.partition(|(_, f)| !collapsed(f));
let order = expanded.into_iter().chain(collapsed).map(|(i, _)| i);
control.fill(order);
let repo = Arc::new(repo);
let reader = repo.blob_reader().map(Arc::new);
if let Ok(reader) = &reader {
let _ = control.git.set((repo.clone(), reader.clone()));
}
out.send(JobMsg::Files(Box::new(DiffFiles {
refs,
files: files.clone(),
generated,
})));
let files = Arc::new(files);
let first: HashSet<usize> = control.front().into_iter().collect();
let (stage_tx, stage_rx) = watch::channel(0u8);
let mut tasks = JoinSet::new();
{
let repo = repo.clone();
let files = files.clone();
let first = first.clone();
tasks.spawn(async move {
let oids = |pick: &dyn Fn(usize) -> bool| -> Vec<String> {
files
.iter()
.enumerate()
.filter(|(i, _)| pick(*i))
.flat_map(|(_, f)| f.blob_oids().map(str::to_owned).collect::<Vec<_>>())
.collect()
};
for (stage, batch) in [(1u8, oids(&|i| first.contains(&i))), (2, oids(&|_| true))] {
match repo.prefetch(&batch).await {
Ok(n) if n > 0 => tracing::info!(n, stage, "prefetched blobs"),
Ok(_) => {}
Err(err) => tracing::warn!(%err, "blob prefetch failed"),
}
let _ = stage_tx.send(stage);
}
});
}
let reader = reader?;
for _ in 0..workers() {
let control = control.clone();
let files = files.clone();
let reader = reader.clone();
let out = out.clone();
let first = first.clone();
let mut stage = stage_rx.clone();
tasks.spawn(async move {
while let Some(index) = control.pop() {
let Some(file) = files.get(index) else {
continue;
};
let needed = if first.contains(&index) { 1 } else { 2 };
if stage.wait_for(|s| *s >= needed).await.is_err() {
return;
}
let diff = diff_file(&reader, file).await;
if !out.send(JobMsg::File(index, Arc::new(diff))) {
return;
}
}
});
}
while let Some(done) = tasks.join_next().await {
if let Err(err) = done
&& err.is_panic()
{
std::panic::resume_unwind(err.into_panic());
}
}
Ok(())
}
fn workers() -> usize {
std::thread::available_parallelism().map_or(2, |n| n.get().clamp(2, 4))
}
async fn sides(
reader: &BlobReader,
file: &ChangedFile,
) -> Result<(Option<Vec<u8>>, Option<Vec<u8>>), GitError> {
let names: Vec<&str> = file.blob_oids().collect();
let mut read = reader.read_each(&names).await?.into_iter();
let mut side = |oid: &Oid| {
if oid.is_zero() {
None
} else {
read.next().flatten()
}
};
Ok((side(&file.old_oid), side(&file.new_oid)))
}
async fn diff_file(reader: &BlobReader, file: &ChangedFile) -> FileDiff {
if file.is_submodule() {
let side = |oid: &str| (oid != ZERO_OID).then(|| oid.to_owned());
return FileDiff::submodule(side(&file.old_oid), side(&file.new_oid));
}
let (old, new) = match sides(reader, file).await {
Ok(sides) => sides,
Err(err) => return FileDiff::error(err.to_string()),
};
let sides = [(&file.old_oid, &old), (&file.new_oid, &new)];
if let Some((oid, _)) = sides
.iter()
.find(|(oid, read)| !oid.is_zero() && read.is_none())
{
let short = crate::ui::text::short_sha(oid);
return FileDiff::error(format!("object {short} is missing"));
}
let path = file.path().to_owned();
tokio::task::spawn_blocking(move || FileDiff::compute(&path, old.as_deref(), new.as_deref()))
.await
.unwrap_or_else(|err| FileDiff::error(format!("diff failed: {err}")))
}
pub async fn since_review_hashes(
repo: &Repo,
reader: &BlobReader,
number: u64,
old_head: &str,
) -> Result<HashSet<String>, GitError> {
repo.ensure(&Oid::parse(old_head)?, &|_| {}).await?;
let base = repo
.rev_parse(&format!("refs/ghtui/pr/{number}/base"))
.await?;
let merge_base = repo.merge_base(&base, old_head).await?;
let files = repo.changed_files(&merge_base, old_head).await?;
let oids: Vec<String> = files
.iter()
.flat_map(|f| f.blob_oids().map(str::to_owned).collect::<Vec<_>>())
.collect();
if let Err(err) = repo.prefetch(&oids).await {
tracing::warn!(%err, "prefetch for the earlier diff failed");
}
let files: Vec<ChangedFile> = files.into_iter().filter(|f| !f.is_submodule()).collect();
let diffs = futures::stream::iter(files)
.map(|file| async move {
let (old, new) = sides(reader, &file).await?;
let path = file.path().to_owned();
let plain = move || FileDiff::compute_plain(&path, old.as_deref(), new.as_deref());
let diff = tokio::task::spawn_blocking(plain).await;
Ok::<_, GitError>(diff.ok().map(|d| d.content))
})
.buffer_unordered(workers());
let mut diffs = std::pin::pin!(diffs);
let mut hashes = HashSet::new();
while let Some(content) = diffs.try_next().await? {
if let Some(crate::diff::Content::Text(text)) = content {
let exact = text.alignment(crate::diff::Whitespace::Exact);
hashes.extend(exact.blocks.iter().map(|b| b.hash.clone()));
}
}
Ok(hashes)
}
pub async fn map_outdated(
repo: &Repo,
reader: &BlobReader,
head: &str,
path: &str,
original_commit: &str,
original_line: u32,
) -> Option<u32> {
let original = format!("{original_commit}:{path}");
if let Err(err) = async { repo.ensure(&Oid::parse(original_commit)?, &|_| {}).await }.await {
tracing::info!(%err, original_commit, "original commit unavailable");
return None;
}
let old = reader.read(&original).await.ok().flatten()?;
let new = reader
.read(&format!("{head}:{path}"))
.await
.ok()
.flatten()?;
let (old, new) = (crate::diff::Text::new(&old), crate::diff::Text::new(&new));
crate::diff::anchor::map_line(&old, &new, original_line)
}
#[cfg(test)]
mod tests {
use super::*;
use std::path::Path;
fn git(dir: &Path, args: &[&str]) -> String {
let out = std::process::Command::new("git")
.arg("-C")
.arg(dir)
.args(args)
.env("GIT_CONFIG_GLOBAL", "/dev/null")
.env("GIT_CONFIG_NOSYSTEM", "1")
.env("GIT_AUTHOR_NAME", "T")
.env("GIT_AUTHOR_EMAIL", "t@example.invalid")
.env("GIT_COMMITTER_NAME", "T")
.env("GIT_COMMITTER_EMAIL", "t@example.invalid")
.output()
.unwrap();
assert!(
out.status.success(),
"{args:?}: {}",
String::from_utf8_lossy(&out.stderr)
);
String::from_utf8_lossy(&out.stdout).trim().to_owned()
}
#[test]
fn prioritize_moves_files_to_front_in_order() {
let control = JobControl::default();
control.fill([0, 1, 2, 3, 4]);
control.prioritize(&[3, 1]);
let order: Vec<usize> = std::iter::from_fn(|| control.pop()).collect();
assert_eq!(order, [3, 1, 0, 2, 4]);
control.prioritize(&[9]);
assert_eq!(control.pop(), None);
}
#[tokio::test]
async fn a_merged_prs_job_lists_its_files() {
let tmp = tempfile::tempdir().unwrap();
let origin = tmp.path().join("origin");
std::fs::create_dir_all(&origin).unwrap();
git(&origin, &["init", "-q", "-b", "main"]);
git(&origin, &["config", "uploadpack.allowFilter", "true"]);
git(
&origin,
&["config", "uploadpack.allowAnySHA1InWant", "true"],
);
std::fs::write(origin.join("a.txt"), "a\n").unwrap();
git(&origin, &["add", "."]);
git(&origin, &["commit", "-q", "-m", "base"]);
git(&origin, &["checkout", "-q", "-b", "feature"]);
std::fs::write(origin.join("b.txt"), "b\n").unwrap();
git(&origin, &["add", "."]);
git(&origin, &["commit", "-q", "-m", "b"]);
git(&origin, &["update-ref", "refs/pull/7/head", "feature"]);
git(&origin, &["checkout", "-q", "main"]);
let base = git(&origin, &["rev-parse", "HEAD"]);
git(
&origin,
&["merge", "-q", "--no-ff", "-m", "Merge #7", "feature"],
);
let local = tmp.path().join("local");
let url = format!("file://{}", origin.display());
git(tmp.path(), &["clone", "-q", &url, "local"]);
git(
&local,
&["remote", "set-url", "origin", "https://github.com/o/r.git"],
);
git(
&local,
&[
"config",
&format!("url.{url}.insteadOf"),
"https://github.com/o/r.git",
],
);
let ctx = GitContext {
cache_root: tmp.path().join("cache"),
credentials: Credentials::Ambient,
cwd: local,
};
let (tx, mut rx) = mpsc::unbounded_channel();
let of = DiffOf::Pr(crate::api::model::PrRef::parse("o/r#7").unwrap());
let out = JobTx { tx, of, job: 1 };
let base = PrBase {
branch: "main".into(),
oid: Some(Oid::parse(&base).unwrap()),
};
run(ctx, base, None, out, Arc::default()).await;
let mut listed = None;
while let Ok(msg) = rx.try_recv() {
match msg {
Msg::Diff(_, DiffMsg::Job(_, JobMsg::Files(files))) => listed = Some(files),
Msg::Diff(_, DiffMsg::Job(_, JobMsg::Failed(err))) => panic!("{err}"),
_ => {}
}
}
let files = listed.expect("the job listed files");
let paths: Vec<&str> = files.files.iter().map(ChangedFile::path).collect();
assert_eq!(paths, ["b.txt"]);
}
#[tokio::test]
async fn outdated_lines_map_to_where_they_moved() {
let lines = |n: u32| (1..=n).map(|i| format!("line {i}\n")).collect::<String>();
let tmp = tempfile::tempdir().unwrap();
let origin = tmp.path().join("origin");
std::fs::create_dir_all(&origin).unwrap();
git(&origin, &["init", "-q", "-b", "main"]);
git(&origin, &["config", "uploadpack.allowFilter", "true"]);
git(
&origin,
&["config", "uploadpack.allowAnySHA1InWant", "true"],
);
std::fs::write(origin.join("a.txt"), lines(20)).unwrap();
git(&origin, &["add", "."]);
git(&origin, &["commit", "-q", "-m", "base"]);
git(&origin, &["checkout", "-q", "-b", "feature"]);
std::fs::write(
origin.join("a.txt"),
lines(20).replace("line 3\n", "three\n"),
)
.unwrap();
git(&origin, &["commit", "-qam", "v1"]);
let written_on = git(&origin, &["rev-parse", "HEAD"]);
git(&origin, &["reset", "-q", "--hard", "main"]);
let v2 = format!(
"new 1\nnew 2\n{}",
lines(20)
.replace("line 3\n", "three\n")
.replace("line 15\n", "fifteen\n")
);
std::fs::write(origin.join("a.txt"), v2).unwrap();
git(&origin, &["commit", "-qam", "v2"]);
git(&origin, &["update-ref", "refs/pull/1/head", "feature"]);
let repo = Repo::open_cache(
&tmp.path().join("cache"),
"o",
"r",
&format!("file://{}", origin.display()),
Credentials::Ambient,
&|_| {},
)
.await
.unwrap();
let head = repo.fetch_pr(1, "main", None, &|_| {}).await.unwrap().head;
let reader = repo.blob_reader().unwrap();
let map = |line| map_outdated(&repo, &reader, &head, "a.txt", &written_on, line);
assert_eq!(map(10).await, Some(12));
assert_eq!(map(3).await, Some(5), "\"three\" is unchanged since");
assert_eq!(map(15).await, None, "line 15 changed");
assert_eq!(
map_outdated(&repo, &reader, &head, "a.txt", &"f".repeat(40), 1).await,
None
);
}
#[tokio::test]
async fn since_review_survives_force_push_and_rebase() {
let numbered = |n: u32| (1..=n).map(|i| format!("line {i}\n")).collect::<String>();
let tmp = tempfile::tempdir().unwrap();
let origin = tmp.path().join("origin");
std::fs::create_dir_all(&origin).unwrap();
git(&origin, &["init", "-q", "-b", "main"]);
git(&origin, &["config", "uploadpack.allowFilter", "true"]);
git(
&origin,
&["config", "uploadpack.allowAnySHA1InWant", "true"],
);
std::fs::write(origin.join("a.txt"), numbered(60)).unwrap();
git(&origin, &["add", "."]);
git(&origin, &["commit", "-q", "-m", "base"]);
git(&origin, &["checkout", "-q", "-b", "feature"]);
std::fs::write(
origin.join("a.txt"),
numbered(60).replace("line 5\n", "five\n"),
)
.unwrap();
git(&origin, &["commit", "-qam", "v1"]);
git(&origin, &["update-ref", "refs/pull/1/head", "feature"]);
let repo = Repo::open_cache(
&tmp.path().join("cache"),
"o",
"r",
&format!("file://{}", origin.display()),
Credentials::Ambient,
&|_| {},
)
.await
.unwrap();
let reviewed = repo.fetch_pr(1, "main", None, &|_| {}).await.unwrap().head;
git(&origin, &["checkout", "-q", "main"]);
std::fs::write(
origin.join("a.txt"),
numbered(60).replace("line 30\n", "thirty\n"),
)
.unwrap();
git(&origin, &["commit", "-qam", "upstream"]);
git(&origin, &["checkout", "-q", "-B", "feature", "main"]);
let rebased = numbered(60)
.replace("line 30\n", "thirty\n")
.replace("line 5\n", "five\n")
.replace("line 55\n", "fifty-five\n");
std::fs::write(origin.join("a.txt"), rebased).unwrap();
git(&origin, &["commit", "-qam", "v2"]);
git(&origin, &["update-ref", "refs/pull/1/head", "feature"]);
let now = repo.fetch_pr(1, "main", None, &|_| {}).await.unwrap();
assert_ne!(now.head, reviewed);
let reader = repo.blob_reader().unwrap();
let seen = since_review_hashes(&repo, &reader, 1, &reviewed)
.await
.unwrap();
let files = repo
.changed_files(&now.merge_base, &now.head)
.await
.unwrap();
assert_eq!(files.len(), 1);
let old = reader.read(&files[0].old_oid).await.unwrap().unwrap();
let new = reader.read(&files[0].new_oid).await.unwrap().unwrap();
let diff = FileDiff::compute_plain("a.txt", Some(&old), Some(&new));
let crate::diff::Content::Text(text) = &diff.content else {
panic!()
};
let exact = text.alignment(crate::diff::Whitespace::Exact);
let blocks = &exact.blocks;
let new_blocks: Vec<String> = blocks
.iter()
.filter(|b| !seen.contains(&b.hash))
.map(|b| {
text.line(exact.lines[b.entries.end as usize - 1].shown())
.to_owned()
})
.collect();
assert_eq!(
blocks.len(),
2,
"upstream's change isn't part of the PR diff"
);
assert_eq!(new_blocks, ["fifty-five"]);
}
}