git_perf/git/

git_interop.rs

use std::{
    io::{BufRead, BufReader, BufWriter, Write},
    path::Path,
    process::Stdio,
    thread,
    time::Duration,
};

use defer::defer;
use log::{debug, warn};
use unindent::unindent;

use anyhow::{anyhow, bail, Context, Result};
use backoff::{ExponentialBackoff, ExponentialBackoffBuilder};
use itertools::Itertools;

use chrono::prelude::*;
use rand::{rng, RngExt};

use crate::config;

pub use super::git_definitions::REFS_NOTES_BRANCH;
use super::git_definitions::{
    GIT_ORIGIN, GIT_PERF_REMOTE, REFS_NOTES_ADD_TARGET_PREFIX, REFS_NOTES_MERGE_BRANCH_PREFIX,
    REFS_NOTES_READ_PREFIX, REFS_NOTES_REWRITE_TARGET_PREFIX, REFS_NOTES_WRITE_SYMBOLIC_REF,
    REFS_NOTES_WRITE_TARGET_PREFIX,
};
use super::git_lowlevel::{
    capture_git_output, get_git_perf_remote, git_rev_parse, git_rev_parse_symbolic_ref,
    git_symbolic_ref_create_or_update, git_update_ref, internal_get_head_revision, is_shallow_repo,
    map_git_error, set_git_perf_remote, spawn_git_command,
};
use super::git_types::GitError;
use super::git_types::GitOutput;
use super::git_types::Reference;

pub use super::git_lowlevel::get_head_revision;

pub use super::git_lowlevel::check_git_version;

pub use super::git_lowlevel::get_repository_root;

pub use super::git_lowlevel::resolve_committish;

/// Represents a commit with its associated git-notes data and metadata.
///
/// This structure is returned by `walk_commits_from` and contains:
/// - The commit SHA
/// - Commit title (subject line)
/// - Author name
/// - Raw note lines for deserialization
#[derive(Debug, Clone, PartialEq)]
pub struct CommitWithNotes {
    pub sha: String,
    pub title: String,
    pub author: String,
    pub note_lines: Vec<String>,
}

/// Check if the current repository is a shallow clone
pub fn is_shallow_repository() -> Result<bool> {
    super::git_lowlevel::is_shallow_repo()
        .map_err(|e| anyhow!("Failed to check if repository is shallow: {}", e))
}

fn map_git_error_for_backoff(e: GitError) -> ::backoff::Error<GitError> {
    match e {
        GitError::RefFailedToPush { .. }
        | GitError::RefFailedToLock { .. }
        | GitError::RefConcurrentModification { .. }
        | GitError::BadObject { .. } => ::backoff::Error::transient(e),
        GitError::ExecError { .. }
        | GitError::IoError(..)
        | GitError::ShallowRepository
        | GitError::MissingHead { .. }
        | GitError::NoRemoteMeasurements { .. }
        | GitError::NoUpstream { .. }
        | GitError::MissingMeasurements => ::backoff::Error::permanent(e),
    }
}

/// Central place to configure backoff policy for git-perf operations.
fn default_backoff() -> ExponentialBackoff {
    let max_elapsed = config::backoff_max_elapsed_seconds();
    ExponentialBackoffBuilder::default()
        .with_max_elapsed_time(Some(Duration::from_secs(max_elapsed)))
        .build()
}

/// Appends a note line to a specific commit with exponential backoff retry logic.
///
/// This function adds a single line to the git notes associated with the specified
/// commit in the performance notes ref (`refs/notes/perf-v3`). The operation is
/// retried with exponential backoff to handle transient failures such as concurrent
/// write conflicts or filesystem locks.
///
/// # Arguments
///
/// * `commit` - The commit hash (or committish reference) to add the note to
/// * `line` - The text content to append to the commit's notes
///
/// # Returns
///
/// * `Ok(())` - The note line was successfully added
/// * `Err` - If the operation fails permanently or times out after retries
///
/// # Errors
///
/// Returns an error if:
/// - The commit does not exist
/// - The operation times out after exhausting retry attempts
/// - A permanent failure occurs (e.g., invalid commit reference)
///
/// # Examples
///
/// ```no_run
/// # use git_perf::git::git_interop::add_note_line;
/// add_note_line("HEAD", "benchmark_result=1.23s").unwrap();
/// ```
pub fn add_note_line(commit: &str, line: &str) -> Result<()> {
    let op = || -> Result<(), ::backoff::Error<GitError>> {
        raw_add_note_line(commit, line).map_err(map_git_error_for_backoff)
    };

    let backoff = default_backoff();

    ::backoff::retry(backoff, op).map_err(|e| match e {
        ::backoff::Error::Permanent(err) => anyhow!(err).context(format!(
            "Permanent failure while adding note line to commit {}",
            commit
        )),
        ::backoff::Error::Transient { err, .. } => anyhow!(err).context(format!(
            "Timed out while adding note line to commit {}",
            commit
        )),
    })?;

    Ok(())
}

/// Add a note line to HEAD (convenience wrapper)
pub fn add_note_line_to_head(line: &str) -> Result<()> {
    let head = internal_get_head_revision()
        .map_err(|e| anyhow!(e).context("Failed to get HEAD revision"))?;
    add_note_line(&head, line)
}

fn raw_add_note_line(commit: &str, line: &str) -> Result<(), GitError> {
    ensure_symbolic_write_ref_exists()?;

    // `git notes append` is not safe to use concurrently.
    // We create a new type of temporary reference: Cannot reuse the normal write references as
    // they only get merged upon push. This can take arbitrarily long.
    let current_note_head =
        git_rev_parse(REFS_NOTES_WRITE_SYMBOLIC_REF).unwrap_or(EMPTY_OID.to_string());
    let current_symbolic_ref_target = git_rev_parse_symbolic_ref(REFS_NOTES_WRITE_SYMBOLIC_REF)
        .expect("Missing symbolic-ref for target");
    let temp_target = create_temp_add_head(&current_note_head)?;

    defer!(remove_reference(&temp_target)
        .expect("Deleting our own temp ref for adding should never fail"));

    // Verify the target commit exists by resolving it
    let resolved_commit = git_rev_parse(commit)?;

    capture_git_output(
        &[
            "notes",
            "--ref",
            &temp_target,
            "append",
            "-m",
            line,
            &resolved_commit,
        ],
        &None,
    )?;

    // Update current write branch with pending write
    // We update the target ref directly (no symref-verify needed in git 2.43.0)
    // The old-oid verification ensures atomicity of the target ref update
    // If the symref was redirected between reading it and updating, the write goes
    // to the old target which will still be merged during consolidation
    git_update_ref(unindent(
        format!(
            r#"
            start
            update {current_symbolic_ref_target} {temp_target} {current_note_head}
            commit
            "#
        )
        .as_str(),
    ))?;

    Ok(())
}

fn ensure_remote_exists() -> Result<(), GitError> {
    if get_git_perf_remote(GIT_PERF_REMOTE).is_some() {
        return Ok(());
    }

    if let Some(x) = get_git_perf_remote(GIT_ORIGIN) {
        return set_git_perf_remote(GIT_PERF_REMOTE, &x);
    }

    Err(GitError::NoUpstream {})
}

/// Creates a temporary reference name by combining a prefix with a random suffix.
fn create_temp_ref_name(prefix: &str) -> String {
    let suffix = random_suffix();
    format!("{prefix}{suffix}")
}

fn ensure_symbolic_write_ref_exists() -> Result<(), GitError> {
    if git_rev_parse(REFS_NOTES_WRITE_SYMBOLIC_REF).is_err() {
        let target = create_temp_ref_name(REFS_NOTES_WRITE_TARGET_PREFIX);

        // Use git symbolic-ref to create the symbolic reference
        // This is not atomic with other ref operations, but that's acceptable
        // as this only runs once during initialization
        git_symbolic_ref_create_or_update(REFS_NOTES_WRITE_SYMBOLIC_REF, &target).or_else(
            |err| {
                // If ref already exists (race with another process), that's fine
                if git_rev_parse(REFS_NOTES_WRITE_SYMBOLIC_REF).is_ok() {
                    Ok(())
                } else {
                    Err(err)
                }
            },
        )?;
    }
    Ok(())
}

fn random_suffix() -> String {
    let suffix: u32 = rng().random::<u32>();
    format!("{suffix:08x}")
}

fn fetch(work_dir: Option<&Path>) -> Result<(), GitError> {
    ensure_remote_exists()?;

    let ref_before = git_rev_parse(REFS_NOTES_BRANCH).ok();
    // Use git directly to avoid having to implement ssh-agent and/or extraHeader handling
    capture_git_output(
        &[
            "fetch",
            "--atomic",
            "--no-write-fetch-head",
            GIT_PERF_REMOTE,
            // Always force overwrite the local reference
            // Separation into write, merge, and read branches ensures that this does not lead to
            // any data loss
            format!("+{REFS_NOTES_BRANCH}:{REFS_NOTES_BRANCH}").as_str(),
        ],
        &work_dir,
    )
    .map_err(map_git_error)?;

    let ref_after = git_rev_parse(REFS_NOTES_BRANCH).ok();

    if ref_before == ref_after {
        println!("Already up to date");
    }

    Ok(())
}

/// Merges notes from one branch into a target using the cat_sort_uniq strategy.
/// This is used to consolidate measurements from multiple write refs.
fn reconcile_branch_with(target: &str, branch: &str) -> Result<(), GitError> {
    _ = capture_git_output(
        &[
            "notes",
            "--ref",
            target,
            "merge",
            "-s",
            "cat_sort_uniq",
            branch,
        ],
        &None,
    )?;
    Ok(())
}

fn create_temp_ref(prefix: &str, current_head: &str) -> Result<String, GitError> {
    let target = create_temp_ref_name(prefix);
    if current_head != EMPTY_OID {
        git_update_ref(unindent(
            format!(
                r#"
            start
            create {target} {current_head}
            commit
            "#
            )
            .as_str(),
        ))?;
    }
    Ok(target)
}

fn create_temp_rewrite_head(current_notes_head: &str) -> Result<String, GitError> {
    create_temp_ref(REFS_NOTES_REWRITE_TARGET_PREFIX, current_notes_head)
}

fn create_temp_add_head(current_notes_head: &str) -> Result<String, GitError> {
    create_temp_ref(REFS_NOTES_ADD_TARGET_PREFIX, current_notes_head)
}

fn compact_head(target: &str) -> Result<(), GitError> {
    let new_removal_head = git_rev_parse(format!("{target}^{{tree}}").as_str())?;

    // Orphan compaction commit
    let compaction_head = capture_git_output(
        &["commit-tree", "-m", "cutoff history", &new_removal_head],
        &None,
    )?
    .stdout;

    let compaction_head = compaction_head.trim();

    git_update_ref(unindent(
        format!(
            r#"
            start
            update {target} {compaction_head}
            commit
            "#
        )
        .as_str(),
    ))?;

    Ok(())
}

fn retry_notify(err: GitError, dur: Duration) {
    debug!("Error happened at {dur:?}: {err}");
    warn!("Retrying...");
}

pub fn remove_measurements_from_commits(
    older_than: DateTime<Utc>,
    prune: bool,
    dry_run: bool,
) -> Result<()> {
    if dry_run {
        // In dry-run mode, don't use backoff retry since we're not modifying anything
        return raw_remove_measurements_from_commits(older_than, prune, dry_run)
            .map_err(|e| anyhow!(e));
    }

    let op = || -> Result<(), ::backoff::Error<GitError>> {
        raw_remove_measurements_from_commits(older_than, prune, dry_run)
            .map_err(map_git_error_for_backoff)
    };

    let backoff = default_backoff();

    ::backoff::retry_notify(backoff, op, retry_notify).map_err(|e| match e {
        ::backoff::Error::Permanent(err) => {
            anyhow!(err).context("Permanent failure while removing measurements")
        }
        ::backoff::Error::Transient { err, .. } => {
            anyhow!(err).context("Timed out while removing measurements")
        }
    })?;

    Ok(())
}

fn execute_notes_operation<F>(operation: F) -> Result<(), GitError>
where
    F: FnOnce(&str) -> Result<(), GitError>,
{
    pull_internal(None)?;

    let current_notes_head = git_rev_parse(REFS_NOTES_BRANCH)?;
    let target = create_temp_rewrite_head(&current_notes_head)?;

    operation(&target)?;

    compact_head(&target)?;

    git_push_notes_ref(&current_notes_head, &target, &None, None)?;

    git_update_ref(unindent(
        format!(
            r#"
            start
            update {REFS_NOTES_BRANCH} {target}
            commit
            "#
        )
        .as_str(),
    ))?;

    remove_reference(&target)?;

    Ok(())
}

fn raw_remove_measurements_from_commits(
    older_than: DateTime<Utc>,
    prune: bool,
    dry_run: bool,
) -> Result<(), GitError> {
    // Check for shallow repo once at the beginning (needed for prune)
    if prune && is_shallow_repo()? {
        return Err(GitError::ShallowRepository);
    }

    if dry_run {
        // In dry-run mode, skip the execute_notes_operation wrapper since we don't modify anything
        remove_measurements_from_reference(REFS_NOTES_BRANCH, older_than, dry_run)?;
        if prune {
            println!("[DRY-RUN] Would prune orphaned measurements after removal");
        }
        return Ok(());
    }

    execute_notes_operation(|target| {
        // Remove measurements older than the specified date
        remove_measurements_from_reference(target, older_than, dry_run)?;

        // Prune orphaned measurements if requested
        if prune {
            capture_git_output(&["notes", "--ref", target, "prune"], &None).map(|_| ())?;
        }

        Ok(())
    })
}

// Remove notes pertaining to git commits whose commit date is older than specified.
fn remove_measurements_from_reference(
    reference: &str,
    older_than: DateTime<Utc>,
    dry_run: bool,
) -> Result<(), GitError> {
    let oldest_timestamp = older_than.timestamp();
    // Outputs line-by-line <note_oid> <annotated_oid>
    let mut list_notes = spawn_git_command(&["notes", "--ref", reference, "list"], &None, None)?;
    let notes_out = list_notes.stdout.take().unwrap();

    let mut get_commit_dates = spawn_git_command(
        &[
            "log",
            "--ignore-missing",
            "--no-walk",
            "--pretty=format:%H %ct",
            "--stdin",
        ],
        &None,
        Some(Stdio::piped()),
    )?;
    let dates_in = get_commit_dates.stdin.take().unwrap();
    let dates_out = get_commit_dates.stdout.take().unwrap();

    if dry_run {
        // In dry-run mode, collect and display what would be removed without actually removing
        let date_collection_handler = thread::spawn(move || {
            let reader = BufReader::new(dates_out);
            let mut results = Vec::new();
            for line in reader.lines().map_while(Result::ok) {
                if let Some((commit, timestamp)) = line.split_whitespace().take(2).collect_tuple() {
                    if let Ok(timestamp) = timestamp.parse::<i64>() {
                        if timestamp <= oldest_timestamp {
                            results.push(commit.to_string());
                        }
                    }
                }
            }
            results
        });

        {
            let reader = BufReader::new(notes_out);
            let mut writer = BufWriter::new(dates_in);

            reader.lines().map_while(Result::ok).for_each(|line| {
                if let Some(line) = line.split_whitespace().nth(1) {
                    writeln!(writer, "{line}").expect("Failed to write to pipe");
                }
            });
        }

        let commits_to_remove = date_collection_handler
            .join()
            .expect("Failed to join date collection thread");
        let count = commits_to_remove.len();

        list_notes.wait()?;
        get_commit_dates.wait()?;

        if count == 0 {
            println!(
                "[DRY-RUN] No measurements older than {} would be removed",
                older_than
            );
        } else {
            println!(
                "[DRY-RUN] Would remove measurements from {} commits older than {}",
                count, older_than
            );
            for commit in &commits_to_remove {
                println!("  {}", commit);
            }
        }

        return Ok(());
    }

    // Normal mode: actually remove measurements
    let mut remove_measurements = spawn_git_command(
        &[
            "notes",
            "--ref",
            reference,
            "remove",
            "--stdin",
            "--ignore-missing",
        ],
        &None,
        Some(Stdio::piped()),
    )?;
    let removal_in = remove_measurements.stdin.take().unwrap();
    let removal_out = remove_measurements.stdout.take().unwrap();

    let removal_handler = thread::spawn(move || {
        let reader = BufReader::new(dates_out);
        let mut writer = BufWriter::new(removal_in);
        for line in reader.lines().map_while(Result::ok) {
            if let Some((commit, timestamp)) = line.split_whitespace().take(2).collect_tuple() {
                if let Ok(timestamp) = timestamp.parse::<i64>() {
                    if timestamp <= oldest_timestamp {
                        writeln!(writer, "{commit}").expect("Could not write to stream");
                    }
                }
            }
        }
    });

    let debugging_handler = thread::spawn(move || {
        let reader = BufReader::new(removal_out);
        reader
            .lines()
            .map_while(Result::ok)
            .for_each(|l| println!("{l}"))
    });

    {
        let reader = BufReader::new(notes_out);
        let mut writer = BufWriter::new(dates_in);

        reader.lines().map_while(Result::ok).for_each(|line| {
            if let Some(line) = line.split_whitespace().nth(1) {
                writeln!(writer, "{line}").expect("Failed to write to pipe");
            }
        });
    }

    removal_handler.join().expect("Failed to join");
    debugging_handler.join().expect("Failed to join");

    list_notes.wait()?;
    get_commit_dates.wait()?;
    remove_measurements.wait()?;

    Ok(())
}

/// Creates a new write ref and updates the symbolic ref to point to it.
/// This is used to ensure concurrent writes go to a new location, preventing
/// race conditions during operations like reset or push.
/// Internal version that returns GitError.
fn new_symbolic_write_ref() -> Result<String, GitError> {
    let target = create_temp_ref_name(REFS_NOTES_WRITE_TARGET_PREFIX);

    // Use git symbolic-ref to update the symbolic reference target
    // This is not atomic with other ref operations, but any concurrent writes
    // that go to the old target will still be merged during consolidation
    git_symbolic_ref_create_or_update(REFS_NOTES_WRITE_SYMBOLIC_REF, &target)?;
    Ok(target)
}

/// Creates a new write ref and updates the symbolic ref to point to it (public wrapper).
/// This is used to ensure concurrent writes go to a new location, preventing
/// race conditions during operations like reset or push.
pub fn create_new_write_ref() -> Result<String> {
    new_symbolic_write_ref().map_err(|e| anyhow!("{:?}", e))
}

const EMPTY_OID: &str = "0000000000000000000000000000000000000000";

fn consolidate_write_branches_into(
    current_upstream_oid: &str,
    target: &str,
    except_ref: Option<&str>,
) -> Result<Vec<Reference>, GitError> {
    // - Reset the merge ref to the upstream perf ref iff it still matches the captured OID
    //   - otherwise concurrent pull occurred.
    git_update_ref(unindent(
        format!(
            r#"
                start
                verify {REFS_NOTES_BRANCH} {current_upstream_oid}
                update {target} {current_upstream_oid} {EMPTY_OID}
                commit
            "#
        )
        .as_str(),
    ))?;

    // - merge in all existing write refs, except for the newly created one from first step
    //     - Same step (except for filtering of the new ref) happens on local read as well.)
    //     - Relies on unrelated histories, cat_sort_uniq merge strategy
    //     - Allows to cut off the history on upstream periodically
    let additional_args = vec![format!("{REFS_NOTES_WRITE_TARGET_PREFIX}*")];
    let refs = get_refs(additional_args)?
        .into_iter()
        .filter(|r| r.refname != except_ref.unwrap_or_default())
        .collect_vec();

    for reference in &refs {
        reconcile_branch_with(target, &reference.oid)?;
    }

    Ok(refs)
}

fn remove_reference(ref_name: &str) -> Result<(), GitError> {
    git_update_ref(unindent(
        format!(
            r#"
                    start
                    delete {ref_name}
                    commit
                "#
        )
        .as_str(),
    ))
}

fn raw_push(work_dir: Option<&Path>, remote: Option<&str>) -> Result<(), GitError> {
    ensure_remote_exists()?;
    // This might merge concurrently created write branches. There is no protection against that.
    // This wants to achieve an at-least-once semantic. The exactly-once semantic is ensured by the
    // cat_sort_uniq merge strategy.

    // - Reset the symbolic-ref "write" to a new unique write ref.
    //     - Allows to continue committing measurements while pushing.
    //     - ?? What happens when a git notes amend concurrently still writes to the old ref?
    let new_write_ref = new_symbolic_write_ref()?;

    let merge_ref = create_temp_ref_name(REFS_NOTES_MERGE_BRANCH_PREFIX);

    defer!(remove_reference(&merge_ref).expect("Deleting our own branch should never fail"));

    // - Create a temporary merge ref, set to the upstream perf ref, merge in all existing write refs except the newly created one from the previous step.
    //     - Same step (except for filtering of the new ref) happens on local read as well.)
    //     - Relies on unrelated histories, cat_sort_uniq merge strategy
    //     - Allows to cut off the history on upstream periodically
    // NEW
    // - Note down the current upstream perf ref oid
    let current_upstream_oid = git_rev_parse(REFS_NOTES_BRANCH).unwrap_or(EMPTY_OID.to_string());
    let refs =
        consolidate_write_branches_into(&current_upstream_oid, &merge_ref, Some(&new_write_ref))?;

    if refs.is_empty() && current_upstream_oid == EMPTY_OID {
        return Err(GitError::MissingMeasurements);
    }

    git_push_notes_ref(&current_upstream_oid, &merge_ref, &work_dir, remote)?;

    // It is acceptable to fetch here independent of the push. Only one concurrent push will succeed.
    fetch(None)?;

    // Delete merged-in write references
    let mut commands = Vec::new();
    commands.push(String::from("start"));
    for Reference { refname, oid } in &refs {
        commands.push(format!("delete {refname} {oid}"));
    }
    commands.push(String::from("commit"));
    // empty line
    commands.push(String::new());
    let commands = commands.join("\n");
    git_update_ref(commands)?;

    Ok(())
}

fn git_push_notes_ref(
    expected_upstream: &str,
    push_ref: &str,
    working_dir: &Option<&Path>,
    remote: Option<&str>,
) -> Result<(), GitError> {
    // - CAS push the temporary merge ref to upstream using the noted down upstream ref
    //     - In case of concurrent pushes, back off and restart fresh from previous step.
    let remote_name = remote.unwrap_or(GIT_PERF_REMOTE);
    let output = capture_git_output(
        &[
            "push",
            "--porcelain",
            format!("--force-with-lease={REFS_NOTES_BRANCH}:{expected_upstream}").as_str(),
            remote_name,
            format!("{push_ref}:{REFS_NOTES_BRANCH}").as_str(),
        ],
        working_dir,
    );

    // - Clean your own temporary merge ref and all others with a merge commit older than x days.
    //     - In case of crashes before clean up, old merge refs are eliminated eventually.

    match output {
        Ok(output) => {
            print!("{}", &output.stdout);
            Ok(())
        }
        Err(GitError::ExecError { output, .. }) => {
            let successful_push = output.stdout.lines().any(|l| {
                l.contains(format!("{REFS_NOTES_BRANCH}:").as_str()) && !l.starts_with('!')
            });
            if successful_push {
                Ok(())
            } else {
                Err(GitError::RefFailedToPush { output })
            }
        }
        Err(e) => Err(e),
    }?;

    Ok(())
}

pub fn prune() -> Result<()> {
    let op = || -> Result<(), ::backoff::Error<GitError>> {
        raw_prune().map_err(map_git_error_for_backoff)
    };

    let backoff = default_backoff();

    ::backoff::retry_notify(backoff, op, retry_notify).map_err(|e| match e {
        ::backoff::Error::Permanent(err) => {
            anyhow!(err).context("Permanent failure while pruning refs")
        }
        ::backoff::Error::Transient { err, .. } => anyhow!(err).context("Timed out pushing refs"),
    })?;

    Ok(())
}

fn raw_prune() -> Result<(), GitError> {
    if is_shallow_repo()? {
        return Err(GitError::ShallowRepository);
    }

    execute_notes_operation(|target| {
        capture_git_output(&["notes", "--ref", target, "prune"], &None).map(|_| ())
    })
}

/// Returns a list of all commit SHA-1 hashes that have performance measurements
/// in the refs/notes/perf-v3 branch.
///
/// Each commit hash is returned as a 40-character hexadecimal string.
pub fn list_commits_with_measurements() -> Result<Vec<String>> {
    // Update local read branch to include pending writes (like walk_commits does)
    let temp_ref = update_read_branch()?;

    // Use git notes list to get all annotated commits
    // Output format: <note_oid> <commit_oid>
    let mut list_notes =
        spawn_git_command(&["notes", "--ref", &temp_ref.ref_name, "list"], &None, None)?;

    let stdout = list_notes
        .stdout
        .take()
        .ok_or_else(|| anyhow!("Failed to capture stdout from git notes list"))?;

    // Parse output line by line: each line is "note_sha commit_sha"
    // We want the commit_sha (second column)
    // Process directly from BufReader for efficiency
    let commits: Vec<String> = BufReader::new(stdout)
        .lines()
        .filter_map(|line_result| {
            line_result
                .ok()
                .and_then(|line| line.split_whitespace().nth(1).map(|s| s.to_string()))
        })
        .collect();

    Ok(commits)
}

/// Guard for a temporary read branch that includes all pending writes.
/// Automatically cleans up the temporary reference when dropped.
pub struct ReadBranchGuard {
    temp_ref: TempRef,
}

impl ReadBranchGuard {
    /// Get the reference name for use in git commands
    #[must_use]
    pub fn ref_name(&self) -> &str {
        &self.temp_ref.ref_name
    }
}

/// Creates a temporary read branch that consolidates all pending writes.
/// The returned guard must be kept alive for as long as the reference is needed.
/// The temporary reference is automatically cleaned up when the guard is dropped.
pub fn create_consolidated_read_branch() -> Result<ReadBranchGuard> {
    let temp_ref = update_read_branch()?;
    Ok(ReadBranchGuard { temp_ref })
}

/// Creates a temporary read branch that consolidates ONLY pending writes (excludes remote).
/// This is used by status and reset commands to see only local pending measurements.
/// The returned guard must be kept alive for as long as the reference is needed.
/// The temporary reference is automatically cleaned up when the guard is dropped.
pub fn create_consolidated_pending_read_branch() -> Result<ReadBranchGuard> {
    let temp_ref = update_pending_read_branch()?;
    Ok(ReadBranchGuard { temp_ref })
}

fn get_refs(additional_args: Vec<String>) -> Result<Vec<Reference>, GitError> {
    let mut args = vec!["for-each-ref", "--format=%(refname)%00%(objectname)"];
    args.extend(additional_args.iter().map(|s| s.as_str()));

    let output = capture_git_output(&args, &None)?;
    let refs: Result<Vec<Reference>, _> = output
        .stdout
        .lines()
        .filter(|s| !s.is_empty())
        .map(|s| {
            let items = s.split('\0').take(2).collect_vec();
            if items.len() != 2 {
                return Err(GitError::ExecError {
                    command: format!("git {}", args.join(" ")),
                    output: GitOutput {
                        stdout: format!("Unexpected git for-each-ref output format: {}", s),
                        stderr: String::new(),
                    },
                });
            }
            Ok(Reference {
                refname: items[0].to_string(),
                oid: items[1].to_string(),
            })
        })
        .collect();
    refs
}

struct TempRef {
    ref_name: String,
}

impl TempRef {
    fn new(prefix: &str) -> Result<Self, GitError> {
        Ok(TempRef {
            ref_name: create_temp_ref(prefix, EMPTY_OID)?,
        })
    }
}

impl Drop for TempRef {
    fn drop(&mut self) {
        remove_reference(&self.ref_name)
            .unwrap_or_else(|_| panic!("Failed to remove reference: {}", self.ref_name))
    }
}

fn update_read_branch() -> Result<TempRef> {
    let temp_ref = TempRef::new(REFS_NOTES_READ_PREFIX)
        .map_err(|e| anyhow!("Failed to create temporary ref: {:?}", e))?;
    // Create a fresh read branch from the remote and consolidate all pending write branches.
    // This ensures the read branch is always up to date with the remote branch, even after
    // a history cutoff, by checking against the current upstream state.
    let current_upstream_oid = git_rev_parse(REFS_NOTES_BRANCH).unwrap_or(EMPTY_OID.to_string());

    consolidate_write_branches_into(&current_upstream_oid, &temp_ref.ref_name, None)
        .map_err(|e| anyhow!("Failed to consolidate write branches: {:?}", e))?;

    Ok(temp_ref)
}

fn update_pending_read_branch() -> Result<TempRef> {
    let temp_ref = TempRef::new(REFS_NOTES_READ_PREFIX)
        .map_err(|e| anyhow!("Failed to create temporary ref: {:?}", e))?;
    // Create a read branch from ONLY the pending write branches (not the remote).
    // Start with empty tree and merge in all write refs.
    let refs = get_refs(vec![format!("{REFS_NOTES_WRITE_TARGET_PREFIX}*")])
        .map_err(|e| anyhow!("Failed to get write refs: {:?}", e))?;

    for reference in &refs {
        reconcile_branch_with(&temp_ref.ref_name, &reference.oid)
            .map_err(|e| anyhow!("Failed to merge write ref: {:?}", e))?;
    }

    Ok(temp_ref)
}

/// Retrieves raw git notes data for commits starting from a specific commit.
///
/// This function performs a low-level git log operation to extract commit hashes
/// and their associated raw note lines from the performance notes ref. It updates
/// the local read branch, resolves the starting commit, and traverses up to
/// `num_commits` following the first-parent ancestry chain.
///
/// # Arguments
///
/// * `start_commit` - The committish reference to start walking from (e.g., "HEAD", "main", commit hash)
/// * `num_commits` - Maximum number of commits to retrieve
///
/// # Returns
///
/// Returns a vector of `CommitWithNotes`, where each entry contains:
/// - The commit SHA-1 hash
/// - The commit title (subject line)
/// - The commit author name
/// - A vector of raw note lines associated with that commit
///
/// # Errors
///
/// Returns an error if:
/// - The starting commit cannot be resolved or does not exist
/// - Git log operation fails
/// - The repository is a shallow clone (issues a warning but may still succeed)
/// - Git log output format is invalid
///
/// # Warnings
///
/// If a shallow clone is detected (grafted commits), a warning is issued as this
/// may result in incomplete history traversal.
///
/// # Examples
///
/// ```no_run
/// # use git_perf::git::git_interop::walk_commits_from;
/// let commits = walk_commits_from("HEAD", 5).unwrap();
/// for commit in commits {
///     println!("Commit: {} by {}: {}", commit.sha, commit.author, commit.title);
/// }
/// ```
pub fn walk_commits_from(start_commit: &str, num_commits: usize) -> Result<Vec<CommitWithNotes>> {
    // update local read branch
    let temp_ref = update_read_branch()?;

    // Resolve and validate the starting commit to ensure it exists
    let resolved_commit = resolve_committish(start_commit)
        .context(format!("Failed to resolve commit '{}'", start_commit))?;

    let output = capture_git_output(
        &[
            "--no-pager",
            "log",
            "--no-color",
            "--ignore-missing",
            "-n",
            num_commits.to_string().as_str(),
            "--first-parent",
            "--pretty=--,%H,%s,%an,%D%n%N",
            "--decorate=full",
            format!("--notes={}", temp_ref.ref_name).as_str(),
            &resolved_commit,
        ],
        &None,
    )
    .context(format!("Failed to retrieve commits from {}", start_commit))?;

    let mut commits: Vec<CommitWithNotes> = Vec::new();
    let mut detected_shallow = false;
    let mut current_commit_sha: Option<String> = None;

    for l in output.stdout.lines() {
        if l.starts_with("--") {
            // Parse format: --,<sha>,<title>,<author>,<decorations>
            let parts: Vec<&str> = l.splitn(5, ',').collect();
            if parts.len() < 5 {
                bail!(
                    "Invalid git log format: expected 5 fields, got {}",
                    parts.len()
                );
            }

            let sha = parts[1].to_string();
            let title = if parts[2].is_empty() {
                "[no subject]".to_string()
            } else {
                parts[2].to_string()
            };
            let author = if parts[3].is_empty() {
                "[unknown]".to_string()
            } else {
                parts[3].to_string()
            };
            let decorations = parts[4];

            detected_shallow |= decorations.contains("grafted");
            current_commit_sha = Some(sha.clone());

            commits.push(CommitWithNotes {
                sha,
                title,
                author,
                note_lines: Vec::new(),
            });
        } else if current_commit_sha.is_some() {
            if let Some(last) = commits.last_mut() {
                last.note_lines.push(l.to_string());
            }
        }
    }

    if detected_shallow && commits.len() < num_commits {
        bail!("Refusing to continue as commit log depth was limited by shallow clone");
    }

    Ok(commits)
}

/// Walk commits starting from HEAD (convenience wrapper)
pub fn walk_commits(num_commits: usize) -> Result<Vec<CommitWithNotes>> {
    walk_commits_from("HEAD", num_commits)
}

/// Get commits that have notes in a specific notes ref.
/// This is much more efficient than walking all commits when you only need
/// commits with measurements.
///
/// Returns a vector of commit SHAs that have notes in the specified ref.
pub fn get_commits_with_notes(notes_ref: &str) -> Result<Vec<String>> {
    let output = capture_git_output(&["notes", "--ref", notes_ref, "list"], &None)
        .context(format!("Failed to list notes in {}", notes_ref))?;

    // git notes list outputs lines in format: <note-sha> <commit-sha>
    let commits: Vec<String> = output
        .stdout
        .lines()
        .filter(|line| !line.is_empty())
        .filter_map(|line| {
            let parts: Vec<&str> = line.split_whitespace().collect();
            if parts.len() >= 2 {
                Some(parts[1].to_string())
            } else {
                None
            }
        })
        .collect();

    Ok(commits)
}

/// Get detailed commit information (SHA, title, author) for specific commits.
/// This is more efficient than walking commits when you already know which commits you need.
pub fn get_commit_details(commit_shas: &[String]) -> Result<Vec<CommitWithNotes>> {
    if commit_shas.is_empty() {
        return Ok(Vec::new());
    }

    let mut commits = Vec::new();

    for sha in commit_shas {
        let output =
            capture_git_output(&["show", "--no-patch", "--format=%H%n%s%n%an", sha], &None)
                .context(format!("Failed to get commit details for {}", sha))?;

        let lines: Vec<&str> = output.stdout.lines().collect();
        if lines.len() >= 3 {
            commits.push(CommitWithNotes {
                sha: lines[0].to_string(),
                title: if lines[1].is_empty() {
                    "[no subject]".to_string()
                } else {
                    lines[1].to_string()
                },
                author: if lines[2].is_empty() {
                    "[unknown]".to_string()
                } else {
                    lines[2].to_string()
                },
                note_lines: Vec::new(), // Will be filled in by caller
            });
        }
    }

    Ok(commits)
}

/// Get the notes content for a specific commit from a notes ref.
/// Returns the note lines as a vector of strings.
pub fn get_notes_for_commit(notes_ref: &str, commit_sha: &str) -> Result<Vec<String>> {
    let output = capture_git_output(&["notes", "--ref", notes_ref, "show", commit_sha], &None);

    match output {
        Ok(output) => {
            let note_lines: Vec<String> = output.stdout.lines().map(|s| s.to_string()).collect();
            Ok(note_lines)
        }
        Err(_) => {
            // No notes for this commit is not an error
            Ok(Vec::new())
        }
    }
}

pub fn pull(work_dir: Option<&Path>) -> Result<()> {
    pull_internal(work_dir)?;
    Ok(())
}

fn pull_internal(work_dir: Option<&Path>) -> Result<(), GitError> {
    fetch(work_dir)?;
    Ok(())
}

pub fn push(work_dir: Option<&Path>, remote: Option<&str>) -> Result<()> {
    let op = || {
        raw_push(work_dir, remote)
            .map_err(map_git_error_for_backoff)
            .map_err(|e: ::backoff::Error<GitError>| match e {
                ::backoff::Error::Transient { .. } => {
                    // Attempt to pull to resolve conflicts
                    let pull_result = pull_internal(work_dir).map_err(map_git_error_for_backoff);

                    // A concurrent modification comes from a concurrent fetch.
                    // Don't fail for that - it's safe to assume we successfully pulled
                    // in the context of the retry logic.
                    let pull_succeeded = pull_result.is_ok()
                        || matches!(
                            pull_result,
                            Err(::backoff::Error::Permanent(
                                GitError::RefConcurrentModification { .. }
                                    | GitError::RefFailedToLock { .. }
                            ))
                        );

                    if pull_succeeded {
                        // Pull succeeded or failed with expected concurrent errors,
                        // return the original push error to retry
                        e
                    } else {
                        // Pull failed with unexpected error, propagate it
                        pull_result.unwrap_err()
                    }
                }
                ::backoff::Error::Permanent { .. } => e,
            })
    };

    let backoff = default_backoff();

    ::backoff::retry_notify(backoff, op, retry_notify).map_err(|e| match e {
        ::backoff::Error::Permanent(err) => {
            anyhow!(err).context("Permanent failure while pushing refs")
        }
        ::backoff::Error::Transient { err, .. } => anyhow!(err).context("Timed out pushing refs"),
    })?;

    Ok(())
}

/// Get all write refs and return their names and OIDs
pub fn get_write_refs() -> Result<Vec<(String, String)>> {
    let refs = get_refs(vec![format!("{REFS_NOTES_WRITE_TARGET_PREFIX}*")])
        .map_err(|e| anyhow!("{:?}", e))?;
    Ok(refs.into_iter().map(|r| (r.refname, r.oid)).collect())
}

/// Delete a git reference (wrapper that converts GitError to anyhow::Error)
pub fn delete_reference(ref_name: &str) -> Result<()> {
    remove_reference(ref_name).map_err(|e| anyhow!("{:?}", e))
}

#[cfg(test)]
mod test {
    use super::*;
    use crate::test_helpers::{run_git_command, with_isolated_cwd_git};
    use std::process::Command;

    use httptest::{
        http::{header::AUTHORIZATION, Uri},
        matchers::{self, request},
        responders::status_code,
        Expectation, Server,
    };

    fn add_server_remote(origin_url: Uri, extra_header: &str, dir: &Path) {
        let url = origin_url.to_string();

        run_git_command(&["remote", "add", "origin", &url], dir);
        run_git_command(
            &[
                "config",
                "--add",
                format!("http.{}.extraHeader", url).as_str(),
                extra_header,
            ],
            dir,
        );
    }

    #[test]
    fn test_customheader_pull() {
        with_isolated_cwd_git(|git_dir| {
            let mut test_server = Server::run();
            add_server_remote(test_server.url(""), "AUTHORIZATION: sometoken", git_dir);

            test_server.expect(
                Expectation::matching(request::headers(matchers::contains((
                    AUTHORIZATION.as_str(),
                    "sometoken",
                ))))
                .times(1..)
                .respond_with(status_code(200)),
            );

            // The pull operation will fail because the mock server doesn't provide a valid git
            // response, but we verify that the authorization header was sent by checking that
            // the server's expectations are met (httptest will panic on drop if not).
            let _ = pull(None); // Ignore result - we only care that auth header was sent

            // Explicitly verify server expectations were met
            test_server.verify_and_clear();
        });
    }

    #[test]
    fn test_customheader_push() {
        with_isolated_cwd_git(|git_dir| {
            let test_server = Server::run();
            add_server_remote(
                test_server.url(""),
                "AUTHORIZATION: someothertoken",
                git_dir,
            );

            test_server.expect(
                Expectation::matching(request::headers(matchers::contains((
                    AUTHORIZATION.as_str(),
                    "someothertoken",
                ))))
                .times(1..)
                .respond_with(status_code(200)),
            );

            // Must add a single write as a push without pending local writes just succeeds
            ensure_symbolic_write_ref_exists().expect("Failed to ensure symbolic write ref exists");
            add_note_line_to_head("test note line").expect("Failed to add note line");

            let error = push(None, None);
            error
                .as_ref()
                .expect_err("We have no valid git http server setup -> should fail");
            dbg!(&error);
        });
    }

    #[test]
    fn test_random_suffix() {
        for _ in 1..1000 {
            let first = random_suffix();
            dbg!(&first);
            let second = random_suffix();
            dbg!(&second);

            let all_hex = |s: &String| s.chars().all(|c| c.is_ascii_hexdigit());

            assert_ne!(first, second);
            assert_eq!(first.len(), 8);
            assert_eq!(second.len(), 8);
            assert!(all_hex(&first));
            assert!(all_hex(&second));
        }
    }

    #[test]
    fn test_empty_or_never_pushed_remote_error_for_fetch() {
        with_isolated_cwd_git(|git_dir| {
            // Add a dummy remote so the code can check for empty remote
            let git_dir_url = format!("file://{}", git_dir.display());
            run_git_command(&["remote", "add", "origin", &git_dir_url], git_dir);

            // NOTE: GIT_TRACE is required for this test to function correctly
            std::env::set_var("GIT_TRACE", "true");

            // Do not add any notes/measurements or push anything
            let result = super::fetch(Some(git_dir));
            match result {
                Err(GitError::NoRemoteMeasurements { output }) => {
                    assert!(
                        output.stderr.contains(GIT_PERF_REMOTE),
                        "Expected output to contain {GIT_PERF_REMOTE}. Output: '{}'",
                        output.stderr
                    )
                }
                other => panic!("Expected NoRemoteMeasurements error, got: {:?}", other),
            }
        });
    }

    #[test]
    fn test_empty_or_never_pushed_remote_error_for_push() {
        with_isolated_cwd_git(|git_dir| {
            run_git_command(&["remote", "add", "origin", "invalid invalid"], git_dir);

            // NOTE: GIT_TRACE is required for this test to function correctly
            std::env::set_var("GIT_TRACE", "true");

            add_note_line_to_head("test line, invalid measurement, does not matter").unwrap();

            let result = super::raw_push(Some(git_dir), None);
            match result {
                Err(GitError::RefFailedToPush { output }) => {
                    assert!(
                        output.stderr.contains(GIT_PERF_REMOTE),
                        "Expected output to contain {GIT_PERF_REMOTE}, got: {}",
                        output.stderr
                    )
                }
                other => panic!("Expected RefFailedToPush error, got: {:?}", other),
            }
        });
    }

    /// Test that new_symbolic_write_ref returns valid, non-empty reference names
    /// Targets missed mutants:
    /// - Could return Ok(String::new()) - empty string
    /// - Could return Ok("xyzzy".into()) - arbitrary invalid string
    #[test]
    fn test_new_symbolic_write_ref_returns_valid_ref() {
        with_isolated_cwd_git(|_git_dir| {
            // Test the private function directly since we're in the same module
            let result = new_symbolic_write_ref();
            assert!(
                result.is_ok(),
                "Should create symbolic write ref: {:?}",
                result
            );

            let ref_name = result.unwrap();

            // Mutation 1: Should not be empty string
            assert!(
                !ref_name.is_empty(),
                "Reference name should not be empty, got: '{}'",
                ref_name
            );

            // Mutation 2: Should not be arbitrary string like "xyzzy"
            assert!(
                ref_name.starts_with(REFS_NOTES_WRITE_TARGET_PREFIX),
                "Reference should start with {}, got: {}",
                REFS_NOTES_WRITE_TARGET_PREFIX,
                ref_name
            );

            // Should have a hex suffix
            let suffix = ref_name
                .strip_prefix(REFS_NOTES_WRITE_TARGET_PREFIX)
                .expect("Should have prefix");
            assert!(
                !suffix.is_empty() && suffix.chars().all(|c| c.is_ascii_hexdigit()),
                "Suffix should be non-empty hex string, got: {}",
                suffix
            );
        });
    }

    /// Test that notes can be added successfully via add_note_line_to_head
    /// Verifies end-to-end note operations work correctly
    #[test]
    fn test_add_and_retrieve_notes() {
        with_isolated_cwd_git(|_git_dir| {
            // Add first note - this calls ensure_symbolic_write_ref_exists -> new_symbolic_write_ref
            let result = add_note_line_to_head("test: 100");
            assert!(
                result.is_ok(),
                "Should add note (requires valid ref from new_symbolic_write_ref): {:?}",
                result
            );

            // Add second note to ensure ref operations continue to work
            let result2 = add_note_line_to_head("test: 200");
            assert!(result2.is_ok(), "Should add second note: {:?}", result2);

            // Verify notes were actually added by walking commits
            let commits = walk_commits(10);
            assert!(commits.is_ok(), "Should walk commits: {:?}", commits);

            let commits = commits.unwrap();
            assert!(!commits.is_empty(), "Should have commits");

            // Check that HEAD commit has notes
            let commit_with_notes = &commits[0];
            assert!(
                !commit_with_notes.note_lines.is_empty(),
                "HEAD should have notes"
            );
            assert!(
                commit_with_notes
                    .note_lines
                    .iter()
                    .any(|n| n.contains("test:")),
                "Notes should contain our test data"
            );
        });
    }

    /// Test walk_commits with shallow repository containing multiple grafted commits
    /// Targets missed mutant at line 725: detected_shallow |= vs ^=
    /// The XOR operator would toggle instead of OR, failing with multiple grafts
    #[test]
    fn test_walk_commits_shallow_repo_detection() {
        use std::env::set_current_dir;

        with_isolated_cwd_git(|git_dir| {
            // Create multiple commits
            for i in 2..=5 {
                run_git_command(
                    &["commit", "--allow-empty", "-m", &format!("Commit {}", i)],
                    git_dir,
                );
            }

            // Create a shallow clone (depth 2) which will have grafted commits
            let shallow_dir = git_dir.join("shallow");
            let output = Command::new("git")
                .args([
                    "clone",
                    "--depth",
                    "2",
                    git_dir.to_str().unwrap(),
                    shallow_dir.to_str().unwrap(),
                ])
                .output()
                .unwrap();

            assert!(
                output.status.success(),
                "Shallow clone failed: {}",
                String::from_utf8_lossy(&output.stderr)
            );

            // Change to shallow clone directory
            set_current_dir(&shallow_dir).unwrap();

            // Add a note to enable walk_commits
            add_note_line_to_head("test: 100").expect("Should add note");

            // Walk commits - should detect as shallow
            let result = walk_commits(10);
            assert!(result.is_ok(), "walk_commits should succeed: {:?}", result);

            let commits = result.unwrap();

            // In a shallow repo, git log --boundary shows grafted markers
            // The |= operator correctly sets detected_shallow to true
            // The ^= mutant would toggle the flag, potentially giving wrong result

            // Verify we got commits (the function works)
            assert!(
                !commits.is_empty(),
                "Should have found commits in shallow repo"
            );
        });
    }

    /// Test walk_commits correctly identifies normal (non-shallow) repos
    #[test]
    fn test_walk_commits_normal_repo_not_shallow() {
        with_isolated_cwd_git(|git_dir| {
            // Create a few commits
            for i in 2..=3 {
                run_git_command(
                    &["commit", "--allow-empty", "-m", &format!("Commit {}", i)],
                    git_dir,
                );
            }

            // Add a note to enable walk_commits
            add_note_line_to_head("test: 100").expect("Should add note");

            let result = walk_commits(10);
            assert!(result.is_ok(), "walk_commits should succeed");

            let commits = result.unwrap();

            // Should have commits
            assert!(!commits.is_empty(), "Should have found commits");
        });
    }
}