git_perf/git/

git_interop.rs

use std::{
    fs::{File, OpenOptions},
    io::{BufRead, BufReader, BufWriter, Write},
    path::Path,
    process::Stdio,
    thread,
    time::Duration,
};

use defer::defer;
use log::{debug, info, warn};
use unindent::unindent;

use anyhow::{anyhow, bail, Context, Result};
use backoff::{ExponentialBackoff, ExponentialBackoffBuilder};
use itertools::Itertools;

use chrono::prelude::*;
use rand::{rng, RngExt};

use crate::config;

pub use super::git_definitions::REFS_NOTES_BRANCH;
use super::git_definitions::{
    GIT_ORIGIN, GIT_PERF_REMOTE, PUSH_LOCK_FILE_NAME, REFS_NOTES_ADD_TARGET_PREFIX,
    REFS_NOTES_MERGE_BRANCH_PREFIX, REFS_NOTES_READ_PREFIX, REFS_NOTES_REWRITE_TARGET_PREFIX,
    REFS_NOTES_WRITE_SYMBOLIC_REF, REFS_NOTES_WRITE_TARGET_PREFIX,
};
use super::git_lowlevel::{
    capture_git_output, get_git_perf_remote, git_common_dir, git_rev_parse,
    git_rev_parse_symbolic_ref, git_symbolic_ref_create_or_update, git_update_ref,
    internal_get_head_revision, is_shallow_repo, map_git_error, set_git_perf_remote,
    spawn_git_command,
};
use super::git_types::GitError;
use super::git_types::GitOutput;
use super::git_types::Reference;

pub use super::git_lowlevel::get_head_revision;

pub use super::git_lowlevel::check_git_version;

pub use super::git_lowlevel::get_repository_root;

pub use super::git_lowlevel::resolve_committish;

/// Represents a commit with its associated git-notes data and metadata.
///
/// This structure is returned by `walk_commits_from` and contains:
/// - The commit SHA
/// - Commit title (subject line)
/// - Author name
/// - Raw note lines for deserialization
#[derive(Debug, Clone, PartialEq)]
pub struct CommitWithNotes {
    pub sha: String,
    pub title: String,
    pub author: String,
    pub note_lines: Vec<String>,
}

/// Check if the current repository is a shallow clone
pub fn is_shallow_repository() -> Result<bool> {
    super::git_lowlevel::is_shallow_repo()
        .map_err(|e| anyhow!("Failed to check if repository is shallow: {}", e))
}

fn map_git_error_for_backoff(e: GitError) -> ::backoff::Error<GitError> {
    match e {
        GitError::RefFailedToPush { .. }
        | GitError::RefFailedToLock { .. }
        | GitError::RefConcurrentModification { .. }
        | GitError::BadObject { .. } => ::backoff::Error::transient(e),
        GitError::ExecError { .. }
        | GitError::IoError(..)
        | GitError::ShallowRepository
        | GitError::MissingHead { .. }
        | GitError::NoRemoteMeasurements { .. }
        | GitError::NoUpstream { .. }
        | GitError::MissingMeasurements => ::backoff::Error::permanent(e),
    }
}

/// Central place to configure backoff policy for git-perf operations.
fn default_backoff() -> ExponentialBackoff {
    let max_elapsed = config::backoff_max_elapsed_seconds();
    ExponentialBackoffBuilder::default()
        .with_max_elapsed_time(Some(Duration::from_secs(max_elapsed)))
        .build()
}

/// Appends a note line to a specific commit with exponential backoff retry logic.
///
/// This function adds a single line to the git notes associated with the specified
/// commit in the performance notes ref (`refs/notes/perf-v3`). The operation is
/// retried with exponential backoff to handle transient failures such as concurrent
/// write conflicts or filesystem locks.
///
/// # Arguments
///
/// * `commit` - The commit hash (or committish reference) to add the note to
/// * `line` - The text content to append to the commit's notes
///
/// # Returns
///
/// * `Ok(())` - The note line was successfully added
/// * `Err` - If the operation fails permanently or times out after retries
///
/// # Errors
///
/// Returns an error if:
/// - The commit does not exist
/// - The operation times out after exhausting retry attempts
/// - A permanent failure occurs (e.g., invalid commit reference)
///
/// # Examples
///
/// ```no_run
/// # use git_perf::git::git_interop::add_note_line;
/// add_note_line("HEAD", "benchmark_result=1.23s").unwrap();
/// ```
pub fn add_note_line(commit: &str, line: &str) -> Result<()> {
    let op = || -> Result<(), ::backoff::Error<GitError>> {
        raw_add_note_line(commit, line).map_err(map_git_error_for_backoff)
    };

    let backoff = default_backoff();

    ::backoff::retry(backoff, op).map_err(|e| match e {
        ::backoff::Error::Permanent(err) => anyhow!(err).context(format!(
            "Permanent failure while adding note line to commit {}",
            commit
        )),
        ::backoff::Error::Transient { err, .. } => anyhow!(err).context(format!(
            "Timed out while adding note line to commit {}",
            commit
        )),
    })?;

    Ok(())
}

/// Add a note line to HEAD (convenience wrapper)
pub fn add_note_line_to_head(line: &str) -> Result<()> {
    let head = internal_get_head_revision()
        .map_err(|e| anyhow!(e).context("Failed to get HEAD revision"))?;
    add_note_line(&head, line)
}

fn raw_add_note_line(commit: &str, line: &str) -> Result<(), GitError> {
    ensure_symbolic_write_ref_exists()?;

    // `git notes append` is not safe to use concurrently.
    // We create a new type of temporary reference: Cannot reuse the normal write references as
    // they only get merged upon push. This can take arbitrarily long.
    let current_note_head =
        git_rev_parse(REFS_NOTES_WRITE_SYMBOLIC_REF).unwrap_or(EMPTY_OID.to_string());
    let current_symbolic_ref_target = git_rev_parse_symbolic_ref(REFS_NOTES_WRITE_SYMBOLIC_REF)
        .ok_or_else(|| GitError::RefFailedToLock {
            output: GitOutput {
                stdout: String::new(),
                stderr: format!(
                    "symbolic-ref {REFS_NOTES_WRITE_SYMBOLIC_REF} vanished after creation"
                ),
            },
        })?;
    let temp_target = create_temp_add_head(&current_note_head)?;

    defer!(if let Err(e) = remove_reference(&temp_target) {
        warn!("Failed to delete temp add ref {temp_target}: {e:?}");
    });

    // Verify the target commit exists by resolving it
    let resolved_commit = git_rev_parse(commit)?;

    capture_git_output(
        &[
            "notes",
            "--ref",
            &temp_target,
            "append",
            "--no-separator",
            "-m",
            line,
            &resolved_commit,
        ],
        &None,
    )
    .map_err(map_git_error)?;

    // Update current write branch with pending write
    // We update the target ref directly (no symref-verify needed in git 2.43.0)
    // The old-oid verification ensures atomicity of the target ref update
    // If the symref was redirected between reading it and updating, the write goes
    // to the old target which will still be merged during consolidation
    git_update_ref(unindent(
        format!(
            r#"
            start
            update {current_symbolic_ref_target} {temp_target} {current_note_head}
            commit
            "#
        )
        .as_str(),
    ))?;

    Ok(())
}

fn ensure_remote_exists() -> Result<(), GitError> {
    if get_git_perf_remote(GIT_PERF_REMOTE).is_some() {
        return Ok(());
    }

    if let Some(x) = get_git_perf_remote(GIT_ORIGIN) {
        return set_git_perf_remote(GIT_PERF_REMOTE, &x);
    }

    Err(GitError::NoUpstream {})
}

/// Creates a temporary reference name by combining a prefix with a random suffix.
fn create_temp_ref_name(prefix: &str) -> String {
    let suffix = random_suffix();
    format!("{prefix}{suffix}")
}

fn ensure_symbolic_write_ref_exists() -> Result<(), GitError> {
    if git_rev_parse(REFS_NOTES_WRITE_SYMBOLIC_REF).is_err() {
        let target = create_temp_ref_name(REFS_NOTES_WRITE_TARGET_PREFIX);

        // Use git symbolic-ref to create the symbolic reference
        // This is not atomic with other ref operations, but that's acceptable
        // as this only runs once during initialization
        git_symbolic_ref_create_or_update(REFS_NOTES_WRITE_SYMBOLIC_REF, &target).or_else(
            |err| {
                // If ref already exists (race with another process), that's fine
                if git_rev_parse(REFS_NOTES_WRITE_SYMBOLIC_REF).is_ok() {
                    Ok(())
                } else {
                    Err(err)
                }
            },
        )?;
    }
    Ok(())
}

fn random_suffix() -> String {
    let pid = std::process::id();
    let random: u32 = rng().random::<u32>();
    format!("{pid:08x}-{random:08x}")
}

fn fetch(work_dir: Option<&Path>) -> Result<(), GitError> {
    ensure_remote_exists()?;

    let ref_before = git_rev_parse(REFS_NOTES_BRANCH).ok();
    // Use git directly to avoid having to implement ssh-agent and/or extraHeader handling
    capture_git_output(
        &[
            "fetch",
            "--atomic",
            "--no-write-fetch-head",
            GIT_PERF_REMOTE,
            // Always force overwrite the local reference
            // Separation into write, merge, and read branches ensures that this does not lead to
            // any data loss
            format!("+{REFS_NOTES_BRANCH}:{REFS_NOTES_BRANCH}").as_str(),
        ],
        &work_dir,
    )
    .map_err(map_git_error)?;

    let ref_after = git_rev_parse(REFS_NOTES_BRANCH).ok();

    if ref_before == ref_after {
        println!("Already up to date");
    }

    Ok(())
}

/// Merges notes from one branch into a target using the cat_sort_uniq strategy.
/// This is used to consolidate measurements from multiple write refs.
fn reconcile_branch_with(target: &str, branch: &str) -> Result<(), GitError> {
    _ = capture_git_output(
        &[
            "notes",
            "--ref",
            target,
            "merge",
            "-s",
            "cat_sort_uniq",
            branch,
        ],
        &None,
    )?;
    Ok(())
}

fn create_temp_ref(prefix: &str, current_head: &str) -> Result<String, GitError> {
    let target = create_temp_ref_name(prefix);
    if current_head != EMPTY_OID {
        git_update_ref(unindent(
            format!(
                r#"
            start
            create {target} {current_head}
            commit
            "#
            )
            .as_str(),
        ))?;
    }
    Ok(target)
}

fn create_temp_rewrite_head(current_notes_head: &str) -> Result<String, GitError> {
    create_temp_ref(REFS_NOTES_REWRITE_TARGET_PREFIX, current_notes_head)
}

fn create_temp_add_head(current_notes_head: &str) -> Result<String, GitError> {
    create_temp_ref(REFS_NOTES_ADD_TARGET_PREFIX, current_notes_head)
}

fn compact_head(target: &str) -> Result<(), GitError> {
    let new_removal_head = git_rev_parse(format!("{target}^{{tree}}").as_str())?;

    // Orphan compaction commit
    let compaction_head = capture_git_output(
        &["commit-tree", "-m", "cutoff history", &new_removal_head],
        &None,
    )?
    .stdout;

    let compaction_head = compaction_head.trim();

    git_update_ref(unindent(
        format!(
            r#"
            start
            update {target} {compaction_head}
            commit
            "#
        )
        .as_str(),
    ))?;

    Ok(())
}

fn retry_notify(err: GitError, dur: Duration) {
    debug!("Error happened at {dur:?}: {err}");
    warn!("Retrying...");
}

pub fn remove_measurements_from_commits(
    older_than: DateTime<Utc>,
    prune: bool,
    dry_run: bool,
) -> Result<()> {
    if dry_run {
        // In dry-run mode, don't use backoff retry since we're not modifying anything
        return raw_remove_measurements_from_commits(older_than, prune, dry_run)
            .map_err(|e| anyhow!(e));
    }

    let op = || -> Result<(), ::backoff::Error<GitError>> {
        raw_remove_measurements_from_commits(older_than, prune, dry_run)
            .map_err(map_git_error_for_backoff)
    };

    let backoff = default_backoff();

    ::backoff::retry_notify(backoff, op, retry_notify).map_err(|e| match e {
        ::backoff::Error::Permanent(err) => {
            anyhow!(err).context("Permanent failure while removing measurements")
        }
        ::backoff::Error::Transient { err, .. } => {
            anyhow!(err).context("Timed out while removing measurements")
        }
    })?;

    Ok(())
}

fn execute_notes_operation<F>(operation: F) -> Result<(), GitError>
where
    F: FnOnce(&str) -> Result<(), GitError>,
{
    pull_internal(None)?;

    let current_notes_head = git_rev_parse(REFS_NOTES_BRANCH)?;
    let target = create_temp_rewrite_head(&current_notes_head)?;

    operation(&target)?;

    // Skip push if the operation made no changes to the notes tree.
    // This avoids unnecessary upstream contention for no-op prune/remove calls.
    if git_rev_parse(&target)? == current_notes_head {
        if let Err(e) = remove_reference(&target) {
            warn!("Failed to delete temp notes ref {target}: {e:?}");
        }
        return Ok(());
    }

    compact_head(&target)?;

    git_push_notes_ref(&current_notes_head, &target, &None, None)?;

    git_update_ref(unindent(
        format!(
            r#"
            start
            update {REFS_NOTES_BRANCH} {target}
            commit
            "#
        )
        .as_str(),
    ))?;

    if let Err(e) = remove_reference(&target) {
        warn!("Failed to delete temp notes ref {target}: {e:?}");
    }

    Ok(())
}

fn raw_remove_measurements_from_commits(
    older_than: DateTime<Utc>,
    prune: bool,
    dry_run: bool,
) -> Result<(), GitError> {
    // Check for shallow repo once at the beginning (needed for prune)
    if prune && is_shallow_repo()? {
        return Err(GitError::ShallowRepository);
    }

    if dry_run {
        // In dry-run mode, skip the execute_notes_operation wrapper since we don't modify anything
        remove_measurements_from_reference(REFS_NOTES_BRANCH, older_than, dry_run)?;
        if prune {
            println!("[DRY-RUN] Would prune orphaned measurements after removal");
        }
        return Ok(());
    }

    execute_notes_operation(|target| {
        // Remove measurements older than the specified date
        remove_measurements_from_reference(target, older_than, dry_run)?;

        // Prune orphaned measurements if requested
        if prune {
            capture_git_output(&["notes", "--ref", target, "prune"], &None).map(|_| ())?;
        }

        Ok(())
    })
}

// Remove notes pertaining to git commits whose commit date is older than specified.
fn remove_measurements_from_reference(
    reference: &str,
    older_than: DateTime<Utc>,
    dry_run: bool,
) -> Result<(), GitError> {
    let oldest_timestamp = older_than.timestamp();
    // Outputs line-by-line <note_oid> <annotated_oid>
    let mut list_notes = spawn_git_command(&["notes", "--ref", reference, "list"], &None, None)?;
    let notes_out = list_notes.stdout.take().unwrap();

    let mut get_commit_dates = spawn_git_command(
        &[
            "log",
            "--ignore-missing",
            "--no-walk",
            "--pretty=format:%H %ct",
            "--stdin",
        ],
        &None,
        Some(Stdio::piped()),
    )?;
    let dates_in = get_commit_dates.stdin.take().unwrap();
    let dates_out = get_commit_dates.stdout.take().unwrap();

    if dry_run {
        // In dry-run mode, collect and display what would be removed without actually removing
        let date_collection_handler = thread::spawn(move || {
            let reader = BufReader::new(dates_out);
            let mut results = Vec::new();
            for line in reader.lines().map_while(Result::ok) {
                if let Some((commit, timestamp)) = line.split_whitespace().take(2).collect_tuple() {
                    if let Ok(timestamp) = timestamp.parse::<i64>() {
                        if timestamp <= oldest_timestamp {
                            results.push(commit.to_string());
                        }
                    }
                }
            }
            results
        });

        {
            let reader = BufReader::new(notes_out);
            let mut writer = BufWriter::new(dates_in);

            reader.lines().map_while(Result::ok).for_each(|line| {
                if let Some(line) = line.split_whitespace().nth(1) {
                    writeln!(writer, "{line}").expect("Failed to write to pipe");
                }
            });
        }

        let commits_to_remove = date_collection_handler
            .join()
            .expect("Failed to join date collection thread");
        let count = commits_to_remove.len();

        list_notes.wait()?;
        get_commit_dates.wait()?;

        if count == 0 {
            println!(
                "[DRY-RUN] No measurements older than {} would be removed",
                older_than
            );
        } else {
            println!(
                "[DRY-RUN] Would remove measurements from {} commits older than {}",
                count, older_than
            );
            for commit in &commits_to_remove {
                println!("  {}", commit);
            }
        }

        return Ok(());
    }

    // Normal mode: actually remove measurements
    let mut remove_measurements = spawn_git_command(
        &[
            "notes",
            "--ref",
            reference,
            "remove",
            "--stdin",
            "--ignore-missing",
        ],
        &None,
        Some(Stdio::piped()),
    )?;
    let removal_in = remove_measurements.stdin.take().unwrap();
    let removal_out = remove_measurements.stdout.take().unwrap();

    let removal_handler = thread::spawn(move || {
        let reader = BufReader::new(dates_out);
        let mut writer = BufWriter::new(removal_in);
        for line in reader.lines().map_while(Result::ok) {
            if let Some((commit, timestamp)) = line.split_whitespace().take(2).collect_tuple() {
                if let Ok(timestamp) = timestamp.parse::<i64>() {
                    if timestamp <= oldest_timestamp {
                        writeln!(writer, "{commit}").expect("Could not write to stream");
                    }
                }
            }
        }
    });

    let debugging_handler = thread::spawn(move || {
        let reader = BufReader::new(removal_out);
        reader
            .lines()
            .map_while(Result::ok)
            .for_each(|l| println!("{l}"))
    });

    {
        let reader = BufReader::new(notes_out);
        let mut writer = BufWriter::new(dates_in);

        reader.lines().map_while(Result::ok).for_each(|line| {
            if let Some(line) = line.split_whitespace().nth(1) {
                writeln!(writer, "{line}").expect("Failed to write to pipe");
            }
        });
    }

    removal_handler.join().expect("Failed to join");
    debugging_handler.join().expect("Failed to join");

    list_notes.wait()?;
    get_commit_dates.wait()?;
    remove_measurements.wait()?;

    Ok(())
}

/// Creates a new write ref and updates the symbolic ref to point to it.
/// This is used to ensure concurrent writes go to a new location, preventing
/// race conditions during operations like reset or push.
/// Internal version that returns GitError.
fn new_symbolic_write_ref() -> Result<String, GitError> {
    let target = create_temp_ref_name(REFS_NOTES_WRITE_TARGET_PREFIX);

    // Use git symbolic-ref to update the symbolic reference target
    // This is not atomic with other ref operations, but any concurrent writes
    // that go to the old target will still be merged during consolidation
    git_symbolic_ref_create_or_update(REFS_NOTES_WRITE_SYMBOLIC_REF, &target)?;
    Ok(target)
}

/// Creates a new write ref and updates the symbolic ref to point to it (public wrapper).
/// This is used to ensure concurrent writes go to a new location, preventing
/// race conditions during operations like reset or push.
pub fn create_new_write_ref() -> Result<String> {
    new_symbolic_write_ref().map_err(|e| anyhow!("{:?}", e))
}

const EMPTY_OID: &str = "0000000000000000000000000000000000000000";

fn consolidate_write_branches_into(
    current_upstream_oid: &str,
    target: &str,
    except_ref: Option<&str>,
) -> Result<Vec<Reference>, GitError> {
    // - Reset the merge ref to the upstream perf ref iff it still matches the captured OID
    //   - otherwise concurrent pull occurred.
    git_update_ref(unindent(
        format!(
            r#"
                start
                verify {REFS_NOTES_BRANCH} {current_upstream_oid}
                update {target} {current_upstream_oid} {EMPTY_OID}
                commit
            "#
        )
        .as_str(),
    ))?;

    // - merge in all existing write refs, except for the newly created one from first step
    //     - Same step (except for filtering of the new ref) happens on local read as well.)
    //     - Relies on unrelated histories, cat_sort_uniq merge strategy
    //     - Allows to cut off the history on upstream periodically
    let additional_args = vec![format!("{REFS_NOTES_WRITE_TARGET_PREFIX}*")];
    let refs = get_refs(additional_args)?
        .into_iter()
        .filter(|r| r.refname != except_ref.unwrap_or_default())
        .collect_vec();

    for reference in &refs {
        reconcile_branch_with(target, &reference.oid)?;
    }

    Ok(refs)
}

fn remove_reference(ref_name: &str) -> Result<(), GitError> {
    git_update_ref(unindent(
        format!(
            r#"
                    start
                    delete {ref_name}
                    commit
                "#
        )
        .as_str(),
    ))
}

fn raw_push(work_dir: Option<&Path>, remote: Option<&str>) -> Result<(), GitError> {
    ensure_remote_exists()?;
    // This might merge concurrently created write branches. There is no protection against that.
    // This wants to achieve an at-least-once semantic. The exactly-once semantic is ensured by the
    // cat_sort_uniq merge strategy.

    // - Reset the symbolic-ref "write" to a new unique write ref.
    //     - Allows to continue committing measurements while pushing.
    //     - ?? What happens when a git notes amend concurrently still writes to the old ref?
    let new_write_ref = new_symbolic_write_ref()?;

    let merge_ref = create_temp_ref_name(REFS_NOTES_MERGE_BRANCH_PREFIX);

    defer!(if let Err(e) = remove_reference(&merge_ref) {
        warn!("Failed to delete temp merge ref {merge_ref}: {e:?}");
    });

    // - Create a temporary merge ref, set to the upstream perf ref, merge in all existing write refs except the newly created one from the previous step.
    //     - Same step (except for filtering of the new ref) happens on local read as well.)
    //     - Relies on unrelated histories, cat_sort_uniq merge strategy
    //     - Allows to cut off the history on upstream periodically
    // NEW
    // - Note down the current upstream perf ref oid
    let current_upstream_oid = git_rev_parse(REFS_NOTES_BRANCH).unwrap_or(EMPTY_OID.to_string());
    let refs =
        consolidate_write_branches_into(&current_upstream_oid, &merge_ref, Some(&new_write_ref))?;

    if refs.is_empty() && current_upstream_oid == EMPTY_OID {
        return Err(GitError::MissingMeasurements);
    }

    git_push_notes_ref(&current_upstream_oid, &merge_ref, &work_dir, remote)?;

    // It is acceptable to fetch here independent of the push. Only one concurrent push will succeed.
    fetch(None)?;

    // Delete merged-in write references
    let mut commands = Vec::new();
    commands.push(String::from("start"));
    for Reference { refname, oid } in &refs {
        commands.push(format!("delete {refname} {oid}"));
    }
    commands.push(String::from("commit"));
    // empty line
    commands.push(String::new());
    let commands = commands.join("\n");
    git_update_ref(commands)?;

    Ok(())
}

/// Acquires an exclusive, blocking advisory lock scoped to this repository.
///
/// Concurrent `git push` invocations from this machine can corrupt each
/// other even though the ref update itself is protected by
/// `--force-with-lease`: each `receive-pack` session gets its own private
/// object quarantine, so a thin pack from one push can delta against an
/// object that only exists in another, still in-flight push's quarantine.
/// The remote's `index-pack` then fails with "unresolved deltas" and the
/// push is rejected — a failure that happens during object transfer, before
/// the CAS check ever runs, so *every* concurrent pusher can lose in the
/// same round instead of exactly one of them winning the ref update. Holding
/// this lock for the duration of a push ensures at most one `git push` to
/// this repository's remotes is ever in flight, restoring the property that
/// the CAS retry loop always makes progress.
///
/// The lock is released automatically when the returned `File` is dropped.
fn acquire_push_lock(working_dir: &Option<&Path>) -> Result<File, GitError> {
    let lock_path = git_common_dir(working_dir)?.join(PUSH_LOCK_FILE_NAME);
    let lock_file = OpenOptions::new()
        .create(true)
        .write(true)
        .truncate(false)
        .open(lock_path)?;
    lock_file.lock()?;
    Ok(lock_file)
}

fn git_push_notes_ref(
    expected_upstream: &str,
    push_ref: &str,
    working_dir: &Option<&Path>,
    remote: Option<&str>,
) -> Result<(), GitError> {
    // - CAS push the temporary merge ref to upstream using the noted down upstream ref
    //     - In case of concurrent pushes, back off and restart fresh from previous step.
    // Serialize the actual push against any other push from this repository -
    // see `acquire_push_lock` for why this is necessary beyond the CAS itself.
    let _push_lock = acquire_push_lock(working_dir)?;

    let remote_name = remote.unwrap_or(GIT_PERF_REMOTE);
    let output = capture_git_output(
        &[
            "push",
            "--porcelain",
            format!("--force-with-lease={REFS_NOTES_BRANCH}:{expected_upstream}").as_str(),
            remote_name,
            format!("{push_ref}:{REFS_NOTES_BRANCH}").as_str(),
        ],
        working_dir,
    );

    // - Clean your own temporary merge ref and all others with a merge commit older than x days.
    //     - In case of crashes before clean up, old merge refs are eliminated eventually.

    match output {
        Ok(output) => {
            print!("{}", output.stdout);
            Ok(())
        }
        Err(GitError::ExecError { output, .. }) => {
            let successful_push = output.stdout.lines().any(|l| {
                l.contains(format!("{REFS_NOTES_BRANCH}:").as_str()) && !l.starts_with('!')
            });
            if successful_push {
                Ok(())
            } else {
                Err(GitError::RefFailedToPush { output })
            }
        }
        Err(e) => Err(e),
    }?;

    Ok(())
}

pub fn prune() -> Result<()> {
    let op = || -> Result<(), ::backoff::Error<GitError>> {
        raw_prune().map_err(map_git_error_for_backoff)
    };

    let backoff = default_backoff();

    ::backoff::retry_notify(backoff, op, retry_notify).map_err(|e| match e {
        ::backoff::Error::Permanent(err) => {
            anyhow!(err).context("Permanent failure while pruning refs")
        }
        ::backoff::Error::Transient { err, .. } => anyhow!(err).context("Timed out pushing refs"),
    })?;

    Ok(())
}

fn extract_pid_from_staging_ref(refname: &str, prefix: &str) -> Option<u32> {
    let suffix = refname.strip_prefix(prefix)?;
    // Old-format refs (8-hex-char only, no dash) predate PID-prefixed naming.
    if !suffix.contains('-') {
        return None;
    }
    let pid_hex = suffix.split('-').next()?;
    u32::from_str_radix(pid_hex, 16).ok()
}

fn is_process_alive(pid: u32) -> bool {
    use sysinfo::{Pid, ProcessRefreshKind, ProcessesToUpdate, System};
    let pid = Pid::from(pid as usize);
    let mut system = System::new();
    system.refresh_processes_specifics(
        ProcessesToUpdate::Some(&[pid]),
        false,
        ProcessRefreshKind::nothing(),
    );
    system.process(pid).is_some()
}

fn cleanup_orphan_staging_refs() {
    // REFS_NOTES_WRITE_TARGET_PREFIX is intentionally excluded: write-target refs accumulate
    // measurements and are consumed by consolidate_write_branches_into() on the next push.
    // Deleting them here would silently discard uncommitted measurements.
    let prefixes = [
        REFS_NOTES_ADD_TARGET_PREFIX,
        REFS_NOTES_MERGE_BRANCH_PREFIX,
        REFS_NOTES_READ_PREFIX,
        REFS_NOTES_REWRITE_TARGET_PREFIX,
    ];
    for prefix in prefixes {
        let refs = get_refs(vec![format!("{prefix}*")]).unwrap_or_default();
        for r in refs {
            let Some(pid) = extract_pid_from_staging_ref(&r.refname, prefix) else {
                // Cannot determine owning process — skip rather than risk deleting a ref
                // from a live process (e.g., old-format refs created before PID tracking).
                continue;
            };
            if is_process_alive(pid) {
                continue;
            }
            info!("Cleaning up orphan staging ref {}", r.refname);
            if let Err(e) = remove_reference(&r.refname) {
                warn!("Failed to clean up orphan ref {}: {e:?}", r.refname);
            }
        }
    }
}

fn raw_prune() -> Result<(), GitError> {
    if is_shallow_repo()? {
        return Err(GitError::ShallowRepository);
    }

    cleanup_orphan_staging_refs();

    execute_notes_operation(|target| {
        capture_git_output(&["notes", "--ref", target, "prune"], &None).map(|_| ())
    })
}

/// Returns a list of all commit SHA-1 hashes that have performance measurements
/// in the refs/notes/perf-v3 branch.
///
/// Each commit hash is returned as a 40-character hexadecimal string.
pub fn list_commits_with_measurements() -> Result<Vec<String>> {
    // Update local read branch to include pending writes (like walk_commits does)
    let temp_ref = update_read_branch()?;

    // Use git notes list to get all annotated commits
    // Output format: <note_oid> <commit_oid>
    let mut list_notes =
        spawn_git_command(&["notes", "--ref", &temp_ref.ref_name, "list"], &None, None)?;

    let stdout = list_notes
        .stdout
        .take()
        .ok_or_else(|| anyhow!("Failed to capture stdout from git notes list"))?;

    // Parse output line by line: each line is "note_sha commit_sha"
    // We want the commit_sha (second column)
    // Process directly from BufReader for efficiency
    let commits: Vec<String> = BufReader::new(stdout)
        .lines()
        .filter_map(|line_result| {
            line_result
                .ok()
                .and_then(|line| line.split_whitespace().nth(1).map(|s| s.to_string()))
        })
        .collect();

    Ok(commits)
}

/// Guard for a temporary read branch that includes all pending writes.
/// Automatically cleans up the temporary reference when dropped.
pub struct ReadBranchGuard {
    temp_ref: TempRef,
}

impl ReadBranchGuard {
    /// Get the reference name for use in git commands
    #[must_use]
    pub fn ref_name(&self) -> &str {
        &self.temp_ref.ref_name
    }
}

/// Creates a temporary read branch that consolidates all pending writes.
/// The returned guard must be kept alive for as long as the reference is needed.
/// The temporary reference is automatically cleaned up when the guard is dropped.
pub fn create_consolidated_read_branch() -> Result<ReadBranchGuard> {
    let temp_ref = update_read_branch()?;
    Ok(ReadBranchGuard { temp_ref })
}

/// Creates a temporary read branch that consolidates ONLY pending writes (excludes remote).
/// This is used by status and reset commands to see only local pending measurements.
/// The returned guard must be kept alive for as long as the reference is needed.
/// The temporary reference is automatically cleaned up when the guard is dropped.
pub fn create_consolidated_pending_read_branch() -> Result<ReadBranchGuard> {
    let temp_ref = update_pending_read_branch()?;
    Ok(ReadBranchGuard { temp_ref })
}

fn get_refs(additional_args: Vec<String>) -> Result<Vec<Reference>, GitError> {
    let mut args = vec!["for-each-ref", "--format=%(refname)%00%(objectname)"];
    args.extend(additional_args.iter().map(|s| s.as_str()));

    let output = capture_git_output(&args, &None)?;
    let refs: Result<Vec<Reference>, _> = output
        .stdout
        .lines()
        .filter(|s| !s.is_empty())
        .map(|s| {
            let items = s.split('\0').take(2).collect_vec();
            if items.len() != 2 {
                return Err(GitError::ExecError {
                    command: format!("git {}", args.join(" ")),
                    output: GitOutput {
                        stdout: format!("Unexpected git for-each-ref output format: {}", s),
                        stderr: String::new(),
                    },
                });
            }
            Ok(Reference {
                refname: items[0].to_string(),
                oid: items[1].to_string(),
            })
        })
        .collect();
    refs
}

struct TempRef {
    ref_name: String,
}

impl TempRef {
    fn new(prefix: &str) -> Result<Self, GitError> {
        Ok(TempRef {
            ref_name: create_temp_ref(prefix, EMPTY_OID)?,
        })
    }
}

impl Drop for TempRef {
    fn drop(&mut self) {
        if let Err(e) = remove_reference(&self.ref_name) {
            warn!("Failed to remove reference {}: {e:?}", self.ref_name);
        }
    }
}

fn update_read_branch() -> Result<TempRef> {
    let temp_ref = TempRef::new(REFS_NOTES_READ_PREFIX)
        .map_err(|e| anyhow!("Failed to create temporary ref: {:?}", e))?;
    // Create a fresh read branch from the remote and consolidate all pending write branches.
    // This ensures the read branch is always up to date with the remote branch, even after
    // a history cutoff, by checking against the current upstream state.
    let current_upstream_oid = git_rev_parse(REFS_NOTES_BRANCH).unwrap_or(EMPTY_OID.to_string());

    consolidate_write_branches_into(&current_upstream_oid, &temp_ref.ref_name, None)
        .map_err(|e| anyhow!("Failed to consolidate write branches: {:?}", e))?;

    Ok(temp_ref)
}

fn update_pending_read_branch() -> Result<TempRef> {
    let temp_ref = TempRef::new(REFS_NOTES_READ_PREFIX)
        .map_err(|e| anyhow!("Failed to create temporary ref: {:?}", e))?;
    // Create a read branch from ONLY the pending write branches (not the remote).
    // Start with empty tree and merge in all write refs.
    let refs = get_refs(vec![format!("{REFS_NOTES_WRITE_TARGET_PREFIX}*")])
        .map_err(|e| anyhow!("Failed to get write refs: {:?}", e))?;

    for reference in &refs {
        reconcile_branch_with(&temp_ref.ref_name, &reference.oid)
            .map_err(|e| anyhow!("Failed to merge write ref: {:?}", e))?;
    }

    Ok(temp_ref)
}

/// Retrieves raw git notes data for commits starting from a specific commit.
///
/// This function performs a low-level git log operation to extract commit hashes
/// and their associated raw note lines from the performance notes ref. It updates
/// the local read branch, resolves the starting commit, and traverses up to
/// `num_commits` following the first-parent ancestry chain.
///
/// # Arguments
///
/// * `start_commit` - The committish reference to start walking from (e.g., "HEAD", "main", commit hash)
/// * `num_commits` - Maximum number of commits to retrieve
///
/// # Returns
///
/// Returns a vector of `CommitWithNotes`, where each entry contains:
/// - The commit SHA-1 hash
/// - The commit title (subject line)
/// - The commit author name
/// - A vector of raw note lines associated with that commit
///
/// # Errors
///
/// Returns an error if:
/// - The starting commit cannot be resolved or does not exist
/// - Git log operation fails
/// - The repository is a shallow clone (issues a warning but may still succeed)
/// - Git log output format is invalid
///
/// # Warnings
///
/// If a shallow clone is detected (grafted commits), a warning is issued as this
/// may result in incomplete history traversal.
///
/// # Examples
///
/// ```no_run
/// # use git_perf::git::git_interop::walk_commits_from;
/// let commits = walk_commits_from("HEAD", 5, None, None).unwrap();
/// for commit in commits {
///     println!("Commit: {} by {}: {}", commit.sha, commit.author, commit.title);
/// }
/// ```
pub fn walk_commits_from(
    start_commit: &str,
    num_commits: usize,
    since: Option<&str>,
    until: Option<&str>,
) -> Result<Vec<CommitWithNotes>> {
    // update local read branch
    let temp_ref = update_read_branch()?;

    // Resolve and validate the starting commit to ensure it exists
    let resolved_commit = resolve_committish(start_commit)
        .context(format!("Failed to resolve commit '{}'", start_commit))?;

    let num_commits_str = num_commits.to_string();
    let notes_ref = format!("--notes={}", temp_ref.ref_name);
    let since_arg = since.map(|s| format!("--since={}", s));
    let until_arg = until.map(|u| format!("--until={}", u));

    let mut args = vec![
        "--no-pager",
        "log",
        "--no-color",
        "--ignore-missing",
        "-n",
        &num_commits_str,
        "--first-parent",
        "--pretty=--,%H,%s,%an,%D%n%N",
        "--decorate=full",
        &notes_ref,
    ];
    if let Some(ref s) = since_arg {
        args.push(s.as_str());
    }
    if let Some(ref u) = until_arg {
        args.push(u.as_str());
    }
    args.push(&resolved_commit);

    let output = capture_git_output(&args, &None)
        .context(format!("Failed to retrieve commits from {}", start_commit))?;

    let mut commits: Vec<CommitWithNotes> = Vec::new();
    let mut detected_shallow = false;
    let mut current_commit_sha: Option<String> = None;

    for l in output.stdout.lines() {
        if l.starts_with("--") {
            // Parse format: --,<sha>,<title>,<author>,<decorations>
            let parts: Vec<&str> = l.splitn(5, ',').collect();
            if parts.len() < 5 {
                bail!(
                    "Invalid git log format: expected 5 fields, got {}",
                    parts.len()
                );
            }

            let sha = parts[1].to_string();
            let title = if parts[2].is_empty() {
                "[no subject]".to_string()
            } else {
                parts[2].to_string()
            };
            let author = if parts[3].is_empty() {
                "[unknown]".to_string()
            } else {
                parts[3].to_string()
            };
            let decorations = parts[4];

            detected_shallow |= decorations.contains("grafted");
            current_commit_sha = Some(sha.clone());

            commits.push(CommitWithNotes {
                sha,
                title,
                author,
                note_lines: Vec::new(),
            });
        } else if current_commit_sha.is_some() {
            if let Some(last) = commits.last_mut() {
                last.note_lines.push(l.to_string());
            }
        }
    }

    if detected_shallow && commits.len() < num_commits {
        bail!("Refusing to continue as commit log depth was limited by shallow clone");
    }

    Ok(commits)
}

/// Walk commits starting from HEAD (convenience wrapper)
pub fn walk_commits(num_commits: usize) -> Result<Vec<CommitWithNotes>> {
    walk_commits_from("HEAD", num_commits, None, None)
}

/// Get commits that have notes in a specific notes ref.
/// This is much more efficient than walking all commits when you only need
/// commits with measurements.
///
/// Returns a vector of commit SHAs that have notes in the specified ref.
pub fn get_commits_with_notes(notes_ref: &str) -> Result<Vec<String>> {
    let output = capture_git_output(&["notes", "--ref", notes_ref, "list"], &None)
        .context(format!("Failed to list notes in {}", notes_ref))?;

    // git notes list outputs lines in format: <note-sha> <commit-sha>
    let commits: Vec<String> = output
        .stdout
        .lines()
        .filter(|line| !line.is_empty())
        .filter_map(|line| {
            let parts: Vec<&str> = line.split_whitespace().collect();
            if parts.len() >= 2 {
                Some(parts[1].to_string())
            } else {
                None
            }
        })
        .collect();

    Ok(commits)
}

/// Batch-fetch all commits that have notes in `notes_ref`, together with their
/// note content, commit subject, and author — in exactly 2 git calls regardless
/// of how many commits have notes.
///
/// Uses `git notes list` to discover which commits have notes, then a single
/// `git log --no-walk` with `--notes=<ref>` to fetch everything in one pass.
pub fn get_commits_with_notes_content(notes_ref: &str) -> Result<Vec<CommitWithNotes>> {
    // Call 1: get commit SHAs that have notes in this ref
    let list_output = capture_git_output(&["notes", "--ref", notes_ref, "list"], &None)
        .context(format!("Failed to list notes in {}", notes_ref))?;

    let commit_shas: Vec<String> = list_output
        .stdout
        .lines()
        .filter(|l| !l.is_empty())
        .filter_map(|line| {
            let parts: Vec<&str> = line.split_whitespace().collect();
            (parts.len() >= 2).then(|| parts[1].to_string())
        })
        .collect();

    if commit_shas.is_empty() {
        return Ok(Vec::new());
    }

    // Call 2: fetch commit metadata + note content in one git log --no-walk pass.
    // Format: each commit header is "<sha>\0<title>\0<author>" (null-separated to
    // handle commas/special chars in subjects), followed by note lines via %N.
    let notes_flag = format!("--notes={}", notes_ref);
    let mut args = vec![
        "--no-pager",
        "log",
        "--no-color",
        "--no-walk",
        "--pretty=format:%H%x00%s%x00%an%n%N",
        &notes_flag,
    ];
    args.extend(commit_shas.iter().map(|s| s.as_str()));

    let output =
        capture_git_output(&args, &None).context("Failed to fetch commit details and notes")?;

    // Parse output: lines containing '\0' are commit headers; all other lines
    // (until the next header) are note content for the preceding commit.
    let mut commits: Vec<CommitWithNotes> = Vec::new();

    for line in output.stdout.lines() {
        if line.contains('\0') {
            let mut parts = line.splitn(3, '\0');
            let sha = parts.next().unwrap_or("").to_string();
            let title = parts
                .next()
                .map(|s| if s.is_empty() { "[no subject]" } else { s })
                .unwrap_or("[no subject]")
                .to_string();
            let author = parts
                .next()
                .map(|s| if s.is_empty() { "[unknown]" } else { s })
                .unwrap_or("[unknown]")
                .to_string();
            commits.push(CommitWithNotes {
                sha,
                title,
                author,
                note_lines: Vec::new(),
            });
        } else if let Some(last) = commits.last_mut() {
            last.note_lines.push(line.to_string());
        }
    }

    Ok(commits)
}

pub fn pull(work_dir: Option<&Path>) -> Result<()> {
    pull_internal(work_dir)?;
    Ok(())
}

fn pull_internal(work_dir: Option<&Path>) -> Result<(), GitError> {
    fetch(work_dir)?;
    Ok(())
}

pub fn push(work_dir: Option<&Path>, remote: Option<&str>) -> Result<()> {
    let op = || {
        raw_push(work_dir, remote)
            .map_err(map_git_error_for_backoff)
            .map_err(|e: ::backoff::Error<GitError>| match e {
                ::backoff::Error::Transient { .. } => {
                    // Attempt to pull to resolve conflicts
                    let pull_result = pull_internal(work_dir).map_err(map_git_error_for_backoff);

                    // A concurrent modification comes from a concurrent fetch.
                    // Don't fail for that - it's safe to assume we successfully pulled
                    // in the context of the retry logic.
                    let pull_succeeded = pull_result.is_ok()
                        || matches!(
                            pull_result,
                            Err(::backoff::Error::Permanent(
                                GitError::RefConcurrentModification { .. }
                                    | GitError::RefFailedToLock { .. }
                            ))
                        );

                    if pull_succeeded {
                        // Pull succeeded or failed with expected concurrent errors,
                        // return the original push error to retry
                        e
                    } else {
                        // Pull failed with unexpected error, propagate it
                        pull_result.unwrap_err()
                    }
                }
                ::backoff::Error::Permanent { .. } => e,
            })
    };

    let backoff = default_backoff();

    ::backoff::retry_notify(backoff, op, retry_notify).map_err(|e| match e {
        ::backoff::Error::Permanent(err) => {
            anyhow!(err).context("Permanent failure while pushing refs")
        }
        ::backoff::Error::Transient { err, .. } => anyhow!(err).context("Timed out pushing refs"),
    })?;

    Ok(())
}

/// Get all write refs and return their names and OIDs
pub fn get_write_refs() -> Result<Vec<(String, String)>> {
    let refs = get_refs(vec![format!("{REFS_NOTES_WRITE_TARGET_PREFIX}*")])
        .map_err(|e| anyhow!("{:?}", e))?;
    Ok(refs.into_iter().map(|r| (r.refname, r.oid)).collect())
}

/// Delete a git reference (wrapper that converts GitError to anyhow::Error)
pub fn delete_reference(ref_name: &str) -> Result<()> {
    remove_reference(ref_name).map_err(|e| anyhow!("{:?}", e))
}

#[cfg(test)]
mod test {
    use super::*;
    use crate::test_helpers::{run_git_command, with_isolated_cwd_git};
    use std::process::Command;
    use std::sync::{
        atomic::{AtomicBool, Ordering},
        Arc,
    };

    use httptest::{
        http::{header::AUTHORIZATION, Uri},
        matchers::{self, request},
        responders::status_code,
        Expectation, Server,
    };

    fn add_server_remote(origin_url: Uri, extra_header: &str, dir: &Path) {
        let url = origin_url.to_string();

        run_git_command(&["remote", "add", "origin", &url], dir);
        run_git_command(
            &[
                "config",
                "--add",
                format!("http.{}.extraHeader", url).as_str(),
                extra_header,
            ],
            dir,
        );
    }

    #[test]
    fn test_customheader_pull() {
        with_isolated_cwd_git(|git_dir| {
            let mut test_server = Server::run();
            add_server_remote(test_server.url(""), "AUTHORIZATION: sometoken", git_dir);

            test_server.expect(
                Expectation::matching(request::headers(matchers::contains((
                    AUTHORIZATION.as_str(),
                    "sometoken",
                ))))
                .times(1..)
                .respond_with(status_code(200)),
            );

            // The pull operation will fail because the mock server doesn't provide a valid git
            // response, but we verify that the authorization header was sent by checking that
            // the server's expectations are met (httptest will panic on drop if not).
            let _ = pull(None); // Ignore result - we only care that auth header was sent

            // Explicitly verify server expectations were met
            test_server.verify_and_clear();
        });
    }

    #[test]
    fn test_customheader_push() {
        with_isolated_cwd_git(|git_dir| {
            let test_server = Server::run();
            add_server_remote(
                test_server.url(""),
                "AUTHORIZATION: someothertoken",
                git_dir,
            );

            test_server.expect(
                Expectation::matching(request::headers(matchers::contains((
                    AUTHORIZATION.as_str(),
                    "someothertoken",
                ))))
                .times(1..)
                .respond_with(status_code(200)),
            );

            // Must add a single write as a push without pending local writes just succeeds
            ensure_symbolic_write_ref_exists().expect("Failed to ensure symbolic write ref exists");
            add_note_line_to_head("test note line").expect("Failed to add note line");

            let error = push(None, None);
            error
                .as_ref()
                .expect_err("We have no valid git http server setup -> should fail");
            dbg!(&error);
        });
    }

    #[test]
    fn test_random_suffix() {
        for _ in 1..1000 {
            let first = random_suffix();
            let second = random_suffix();

            // Format: {pid:08x}-{random:08x}
            let is_valid = |s: &String| {
                let parts: Vec<&str> = s.splitn(2, '-').collect();
                parts.len() == 2
                    && parts[0].len() == 8
                    && parts[0].chars().all(|c| c.is_ascii_hexdigit())
                    && parts[1].len() == 8
                    && parts[1].chars().all(|c| c.is_ascii_hexdigit())
            };

            assert_ne!(first, second);
            assert!(is_valid(&first), "Invalid suffix format: {}", first);
            assert!(is_valid(&second), "Invalid suffix format: {}", second);
        }
    }

    #[test]
    fn test_empty_or_never_pushed_remote_error_for_fetch() {
        with_isolated_cwd_git(|git_dir| {
            // Add a dummy remote so the code can check for empty remote
            let git_dir_url = format!("file://{}", git_dir.display());
            run_git_command(&["remote", "add", "origin", &git_dir_url], git_dir);

            // NOTE: GIT_TRACE is required for this test to function correctly
            std::env::set_var("GIT_TRACE", "true");

            // Do not add any notes/measurements or push anything
            let result = super::fetch(Some(git_dir));
            match result {
                Err(GitError::NoRemoteMeasurements { output }) => {
                    assert!(
                        output.stderr.contains(GIT_PERF_REMOTE),
                        "Expected output to contain {GIT_PERF_REMOTE}. Output: '{}'",
                        output.stderr
                    )
                }
                other => panic!("Expected NoRemoteMeasurements error, got: {:?}", other),
            }
        });
    }

    #[test]
    fn test_empty_or_never_pushed_remote_error_for_push() {
        with_isolated_cwd_git(|git_dir| {
            run_git_command(&["remote", "add", "origin", "invalid invalid"], git_dir);

            // NOTE: GIT_TRACE is required for this test to function correctly
            std::env::set_var("GIT_TRACE", "true");

            add_note_line_to_head("test line, invalid measurement, does not matter").unwrap();

            let result = super::raw_push(Some(git_dir), None);
            match result {
                Err(GitError::RefFailedToPush { output }) => {
                    assert!(
                        output.stderr.contains(GIT_PERF_REMOTE),
                        "Expected output to contain {GIT_PERF_REMOTE}, got: {}",
                        output.stderr
                    )
                }
                other => panic!("Expected RefFailedToPush error, got: {:?}", other),
            }
        });
    }

    /// Verifies the push lock file lives inside the repository's common git
    /// dir, targeting mutants that could point it at an arbitrary location
    /// (which would silently defeat serialization between two repos, or
    /// between a repo and an unrelated directory).
    #[test]
    fn test_acquire_push_lock_creates_lock_file_in_git_common_dir() {
        with_isolated_cwd_git(|git_dir| {
            let lock = acquire_push_lock(&None).expect("should acquire push lock");
            drop(lock);

            let expected_path = git_dir.join(".git").join(PUSH_LOCK_FILE_NAME);
            assert!(
                expected_path.exists(),
                "Expected lock file at {:?}",
                expected_path
            );
        });
    }

    /// Verifies the lock actually excludes concurrent access: a second,
    /// independent file handle cannot acquire it while the first is held,
    /// and can once the first is released. This is the core property the
    /// fix for #732 relies on to prevent two `git push` invocations from
    /// racing inside receive-pack's object quarantine.
    #[test]
    fn test_acquire_push_lock_excludes_concurrent_access() {
        with_isolated_cwd_git(|_git_dir| {
            let first = acquire_push_lock(&None).expect("first lock should succeed");

            let lock_path = git_common_dir(&None)
                .expect("git common dir")
                .join(PUSH_LOCK_FILE_NAME);
            let second = OpenOptions::new()
                .create(true)
                .write(true)
                .truncate(false)
                .open(&lock_path)
                .expect("open lock file for contention probe");
            assert!(
                second.try_lock().is_err(),
                "Second handle should not be able to lock while the first is held"
            );

            drop(first);

            second
                .try_lock()
                .expect("Second handle should acquire the lock once the first is released");
        });
    }

    /// Verifies a blocked contender actually wakes up and proceeds once the
    /// lock is released, rather than deadlocking - the liveness property the
    /// fix needs so that at least one of two concurrent pushers always makes
    /// progress instead of both repeatedly corrupting each other's pack
    /// transfer (see #732 / #745).
    #[test]
    fn test_acquire_push_lock_blocks_until_released() {
        with_isolated_cwd_git(|git_dir| {
            let git_dir = git_dir.to_path_buf();
            let first =
                acquire_push_lock(&Some(git_dir.as_path())).expect("first lock should succeed");

            let acquired = Arc::new(AtomicBool::new(false));
            let acquired_writer = Arc::clone(&acquired);
            let waiter_git_dir = git_dir.clone();
            let waiter = thread::spawn(move || {
                let _second = acquire_push_lock(&Some(waiter_git_dir.as_path()))
                    .expect("second lock should eventually succeed");
                acquired_writer.store(true, Ordering::SeqCst);
            });

            thread::sleep(Duration::from_millis(200));
            assert!(
                !acquired.load(Ordering::SeqCst),
                "Contender should still be blocked while the first lock is held"
            );

            drop(first);
            waiter.join().expect("waiter thread should not panic");
            assert!(
                acquired.load(Ordering::SeqCst),
                "Contender should acquire the lock after it is released"
            );
        });
    }

    /// Test that new_symbolic_write_ref returns valid, non-empty reference names
    /// Targets missed mutants:
    /// - Could return Ok(String::new()) - empty string
    /// - Could return Ok("xyzzy".into()) - arbitrary invalid string
    #[test]
    fn test_new_symbolic_write_ref_returns_valid_ref() {
        with_isolated_cwd_git(|_git_dir| {
            // Test the private function directly since we're in the same module
            let result = new_symbolic_write_ref();
            assert!(
                result.is_ok(),
                "Should create symbolic write ref: {:?}",
                result
            );

            let ref_name = result.unwrap();

            // Mutation 1: Should not be empty string
            assert!(
                !ref_name.is_empty(),
                "Reference name should not be empty, got: '{}'",
                ref_name
            );

            // Mutation 2: Should not be arbitrary string like "xyzzy"
            assert!(
                ref_name.starts_with(REFS_NOTES_WRITE_TARGET_PREFIX),
                "Reference should start with {}, got: {}",
                REFS_NOTES_WRITE_TARGET_PREFIX,
                ref_name
            );

            // Should have a suffix in format {pid:08x}-{random:08x}
            let suffix = ref_name
                .strip_prefix(REFS_NOTES_WRITE_TARGET_PREFIX)
                .expect("Should have prefix");
            let parts: Vec<&str> = suffix.splitn(2, '-').collect();
            let is_valid_suffix = parts.len() == 2
                && parts[0].len() == 8
                && parts[0].chars().all(|c| c.is_ascii_hexdigit())
                && parts[1].len() == 8
                && parts[1].chars().all(|c| c.is_ascii_hexdigit());
            assert!(
                is_valid_suffix,
                "Suffix should be in format {{pid:08x}}-{{random:08x}}, got: {}",
                suffix
            );
        });
    }

    /// Test that notes can be added successfully via add_note_line_to_head
    /// Verifies end-to-end note operations work correctly
    #[test]
    fn test_add_and_retrieve_notes() {
        with_isolated_cwd_git(|_git_dir| {
            // Add first note - this calls ensure_symbolic_write_ref_exists -> new_symbolic_write_ref
            let result = add_note_line_to_head("test: 100");
            assert!(
                result.is_ok(),
                "Should add note (requires valid ref from new_symbolic_write_ref): {:?}",
                result
            );

            // Add second note to ensure ref operations continue to work
            let result2 = add_note_line_to_head("test: 200");
            assert!(result2.is_ok(), "Should add second note: {:?}", result2);

            // Verify notes were actually added by walking commits
            let commits = walk_commits(10);
            assert!(commits.is_ok(), "Should walk commits: {:?}", commits);

            let commits = commits.unwrap();
            assert!(!commits.is_empty(), "Should have commits");

            // Check that HEAD commit has notes
            let commit_with_notes = &commits[0];
            assert!(
                !commit_with_notes.note_lines.is_empty(),
                "HEAD should have notes"
            );
            assert!(
                commit_with_notes
                    .note_lines
                    .iter()
                    .any(|n| n.contains("test:")),
                "Notes should contain our test data"
            );
        });
    }

    /// Test walk_commits with shallow repository containing multiple grafted commits
    /// Targets missed mutant at line 725: detected_shallow |= vs ^=
    /// The XOR operator would toggle instead of OR, failing with multiple grafts
    #[test]
    fn test_walk_commits_shallow_repo_detection() {
        use std::env::set_current_dir;

        with_isolated_cwd_git(|git_dir| {
            // Create multiple commits
            for i in 2..=5 {
                run_git_command(
                    &["commit", "--allow-empty", "-m", &format!("Commit {}", i)],
                    git_dir,
                );
            }

            // Create a shallow clone (depth 2) which will have grafted commits
            let shallow_dir = git_dir.join("shallow");
            let output = Command::new("git")
                .args([
                    "clone",
                    "--depth",
                    "2",
                    git_dir.to_str().unwrap(),
                    shallow_dir.to_str().unwrap(),
                ])
                .output()
                .unwrap();

            assert!(
                output.status.success(),
                "Shallow clone failed: {}",
                String::from_utf8_lossy(&output.stderr)
            );

            // Change to shallow clone directory
            set_current_dir(&shallow_dir).unwrap();

            // Add a note to enable walk_commits
            add_note_line_to_head("test: 100").expect("Should add note");

            // Walk commits - should detect as shallow
            let result = walk_commits(10);
            assert!(result.is_ok(), "walk_commits should succeed: {:?}", result);

            let commits = result.unwrap();

            // In a shallow repo, git log --boundary shows grafted markers
            // The |= operator correctly sets detected_shallow to true
            // The ^= mutant would toggle the flag, potentially giving wrong result

            // Verify we got commits (the function works)
            assert!(
                !commits.is_empty(),
                "Should have found commits in shallow repo"
            );
        });
    }

    /// Test walk_commits correctly identifies normal (non-shallow) repos
    #[test]
    fn test_walk_commits_normal_repo_not_shallow() {
        with_isolated_cwd_git(|git_dir| {
            // Create a few commits
            for i in 2..=3 {
                run_git_command(
                    &["commit", "--allow-empty", "-m", &format!("Commit {}", i)],
                    git_dir,
                );
            }

            // Add a note to enable walk_commits
            add_note_line_to_head("test: 100").expect("Should add note");

            let result = walk_commits(10);
            assert!(result.is_ok(), "walk_commits should succeed");

            let commits = result.unwrap();

            // Should have commits
            assert!(!commits.is_empty(), "Should have found commits");
        });
    }

    #[test]
    fn test_extract_pid_from_staging_ref() {
        // Valid new-format suffix: {pid:08x}-{random:08x}
        let refname = format!("{}{}", REFS_NOTES_ADD_TARGET_PREFIX, "deadbeef-01234567");
        assert_eq!(
            extract_pid_from_staging_ref(&refname, REFS_NOTES_ADD_TARGET_PREFIX),
            Some(0xdeadbeef),
        );

        // Old-format suffix: 8-hex-char only, no dash — must return None
        // (would overflow to negative i32 and call kill(-N, 0) on a process group)
        let old_refname = format!("{}{}", REFS_NOTES_ADD_TARGET_PREFIX, "deadbeef");
        assert_eq!(
            extract_pid_from_staging_ref(&old_refname, REFS_NOTES_ADD_TARGET_PREFIX),
            None,
            "Old-format refs without a dash must return None to avoid kill(-N, 0) on a process group",
        );

        // Wrong prefix — strip_prefix returns None
        let wrong_prefix = format!("{}{}", REFS_NOTES_WRITE_TARGET_PREFIX, "deadbeef-01234567");
        assert_eq!(
            extract_pid_from_staging_ref(&wrong_prefix, REFS_NOTES_ADD_TARGET_PREFIX),
            None,
        );

        // Non-hex characters in pid field — from_str_radix returns Err
        let bad_hex = format!("{}{}", REFS_NOTES_ADD_TARGET_PREFIX, "xyz00000-01234567");
        assert_eq!(
            extract_pid_from_staging_ref(&bad_hex, REFS_NOTES_ADD_TARGET_PREFIX),
            None,
        );
    }

    #[test]
    fn test_is_process_alive_returns_true_for_current_process() {
        assert!(
            is_process_alive(std::process::id()),
            "Current process must be alive",
        );
    }

    #[cfg(unix)]
    #[test]
    fn test_is_process_alive_returns_false_for_dead_process() {
        let mut child = std::process::Command::new("true")
            .spawn()
            .expect("Failed to spawn 'true'");
        let pid = child.id();
        child.wait().expect("Failed to wait for child");
        // PID is freed after wait(); extremely unlikely to be reused before the next line
        assert!(
            !is_process_alive(pid),
            "PID {pid} should be dead after process exited",
        );
    }

    /// Test that raw_add_note_line succeeds under normal conditions (symref present).
    /// This directly exercises the ok_or_else path without the 60-second backoff
    /// wrapper, so any mutant that makes the function always return an error is
    /// caught immediately rather than after a full backoff cycle.
    #[test]
    fn test_raw_add_note_line_succeeds_in_normal_repo() {
        with_isolated_cwd_git(|_git_dir| {
            let result = raw_add_note_line("HEAD", "test_measurement=1.0");
            assert!(
                result.is_ok(),
                "raw_add_note_line should succeed in a freshly initialised repo: {:?}",
                result
            );
        });
    }

    /// Test that execute_notes_operation skips the upstream push when the operation
    /// makes no changes (no-op prune/remove). Targets the early-return branch:
    ///   if git_rev_parse(&target)? == current_notes_head { return Ok(()); }
    /// A mutant that flips `==` to `!=` would reach compact_head and push,
    /// changing the upstream hash — which this test detects.
    #[test]
    fn test_execute_notes_operation_noop_skips_upstream_push() {
        use tempfile::tempdir;

        with_isolated_cwd_git(|git_dir| {
            // Set up a bare upstream repo and add it as origin
            let upstream = tempdir().unwrap();
            run_git_command(&["init", "--bare"], upstream.path());
            let upstream_url = format!("file://{}", upstream.path().display());
            run_git_command(&["remote", "add", "origin", &upstream_url], git_dir);
            run_git_command(&["push", "origin", "master"], git_dir);

            // Add a measurement and push to create the upstream notes branch
            add_note_line_to_head("test: 1").expect("add note");
            push(None, None).expect("push notes");

            let head_before = git_rev_parse(REFS_NOTES_BRANCH).expect("notes branch exists");

            // A closure that does nothing to the target ref is a no-op
            execute_notes_operation(|_target| Ok(()))
                .expect("no-op execute_notes_operation should succeed");

            // The notes branch must be unchanged — no spurious push occurred
            let head_after = git_rev_parse(REFS_NOTES_BRANCH).expect("notes branch still exists");
            assert_eq!(
                head_before, head_after,
                "no-op execute_notes_operation must not advance the notes branch"
            );

            // All temp rewrite refs must be cleaned up
            let rewrite_refs = get_refs(vec![format!("{REFS_NOTES_REWRITE_TARGET_PREFIX}*")])
                .expect("list rewrite refs");
            assert!(
                rewrite_refs.is_empty(),
                "temp rewrite refs must be cleaned up after no-op: {:?}",
                rewrite_refs
            );
        });
    }
}