git_policy_cli/

service_work.rs

//! What:
//!  The service's consumer of after-commit work: pending `push` and `maintenance` work items of one repository are
//!  claimed, coalesced, run, settled, and a push that failed leaves a record that the person's next wrapper command shows.
//! Why:
//!  A commit must not wait for the network or for upkeep, so the wrapper leaves a record and the resident service does
//!  the work on its own thread (`doc/planning/cli-git-resident-service.md`, "After-commit work"). The service is the
//!  only pusher of the repositories it serves, so no lock is needed: several pending pushes of one branch become one
//!  push, and a service that died mid-push is noticed by its successor's first drain, which takes back every claim it finds,
//!  never by a process id. Other kinds of work
//!  (the standards review) are never claimed here.
//!
//! In TS you'd write (pseudocode):
//! ```ts
//! for (const group of coalesce(await claimAll(workDir, ['push', 'maintenance']))) settle(group, await handler(group));
//! ```

use super::auto_push::push_once;
use super::auto_push_single_flight::PushGit;
use super::config_snapshot::read_config_snapshot;
use super::lfs_pre_push::uses_lfs;
use super::maintenance_run::{AutoMaintenanceOutcome, run_auto_maintenance_with};
use super::push_failure_notice::{clear_push_failures, record_push_failure};
use super::repo_read_config_text::{last_entry, parse_bool, parse_config_text};
use super::service_claim_renewal::{ClaimRenewal, HeldClaims, RENEWALS_PER_LEASE};
use super::service_review::{BUDGET, options_for, review_item, review_variables};
use super::service_review_key::ReviewKey;
use super::standards_review_run::ReviewOptions;
use super::transaction_git::GitContext;
use super::work_item::{DONE, FAILED, WORK_DIRECTORY, WorkItem, WorkKind, claim_next_of, renew_claim, requeue_claimed, requeue_expired, settle};
use std::path::{Path, PathBuf};
use std::time::SystemTime;

/// The kinds of work the service runs itself.
pub const SERVICE_KINDS: [WorkKind; 3] = [WorkKind::Push, WorkKind::Maintenance, WorkKind::Review];

/// How many times an item is tried before it is given up on.
pub const MAX_TRIES: u32 = 3;

/// What a wake message brings: the repository to work on and the review credential that came with it, if any.
pub type Wake = (PathBuf, Option<ReviewKey>);

/// What a handler came to.
#[derive(Clone, Debug, Eq, PartialEq)]
pub enum Verdict {
    /// The work was done, or there was nothing to do.
    Done,
    /// The work failed; the text is what the person should be told.
    Failed(String),
}

/// What runs each kind of work.
pub trait WorkHandlers {
    /// Push the branch of these items, which share a repository and a branch; the last is the newest.
    fn push(&self, items: &[WorkItem]) -> Verdict;
    /// Run the repository's automatic upkeep.
    fn maintenance(&self, item: &WorkItem) -> Verdict;
    /// Review one landed commit into the repository's shadow log.
    fn review(&self, item: &WorkItem) -> Verdict;
}

/// The work directory of a repository.
pub fn work_directory(git_dir: &Path) -> PathBuf {
    return git_dir.join(WORK_DIRECTORY);
}

/// What:
///  Whether an item is one this service should act on for this repository.
/// Why:
///   Anyone who can write the work directory can write an item, so the fields that become a directory, a reference or a
///  program argument are checked before they are used.
///
/// In TS you'd write (pseudocode):
/// ```ts
/// isAcceptable(item, gitDir)
/// ```
pub fn is_acceptable(item: &WorkItem, git_dir: &Path) -> bool {
    return Path::new(item.repository.as_str()).is_absolute()
        && Path::new(item.repository.as_str()).is_dir()
        && Path::new(item.git_dir.as_str()) == git_dir
        && item.branch_ref.starts_with("refs/")
        && !item.branch_ref.contains(['\0', '\n', ' '])
        && item.oid.len() >= 40
        && item.oid.bytes().all(|byte| { return byte.is_ascii_hexdigit(); });
}

/// Moves the claims to `state` and stops renewing them.
fn settle_all(work_dir: &Path, held: &HeldClaims, claimed: &[super::work_item::Claimed], state: &str) {
    for item in claimed {
        if let Err(reason) = settle(work_dir, item, state) {
            eprintln!("cli-git-service: {reason}");
        }
        held.release(item.id.as_str());
    }
}

/// What:
///  Whether a common Git directory belongs to a repository with no main worktree, which `core.bare` in its own `config` says.
/// Why:
///   A bare repository has no instance whose Git directory is the common one, so the instances of its linked worktrees are the
///  only ones that can run the records left there; a repository with a main worktree has that instance, which owns them.
///   Only the common file is read: an unreadable or unparsable file, or a spelling this reader leaves to Git, reads as not
///  bare, and the records then stay for the main worktree's instance, which is the safe side of a wrong answer.
///
/// In TS you'd write (pseudocode):
/// ```ts
/// isBareCommonDirectory(commonDir)
/// ```
fn is_bare_common_directory(common: &Path) -> bool {
    let Ok(text) = std::fs::read(common.join("config")) else {
        return false;
    };
    let Some(entries) = parse_config_text(text.as_slice()) else {
        return false;
    };
    return last_entry(entries.as_slice(), "core.bare").is_some_and(|entry| { return parse_bool(entry.value.as_deref()) == Some(true); });
}

/// What:
///  Runs everything pending for one repository: takes back the claims an earlier service left, claims every pending push and maintenance
///  item, runs one push per branch and one upkeep per repository, settles the items, and leaves a notice for each
///  failed push.
/// Why:
///   Coalescing keeps a burst of commits to one push; a handler that fails leaves the items in `failed`, not in a loop.
///
/// In TS you'd write (pseudocode):
/// ```ts
/// await drain(gitDir, handlers, new Date())
/// ```
pub fn drain(git_dir: &Path, handlers: &dyn WorkHandlers, now: SystemTime) {
    let work_dir: PathBuf = work_directory(git_dir);
    if is_bare_common_directory(git_dir) {
        // The linked worktrees' instances drain this directory too, so a claim here may be live: only its lease says otherwise.
        let _requeued = requeue_expired(work_dir.as_path(), now, SHARED_CLAIM_LEASE, MAX_TRIES);
        drain_pending(git_dir, handlers, now, Some(SHARED_CLAIM_LEASE));
        return;
    }
    // This thread claims nothing before this line, and one service serves a worktree at a time, so a claim found in its own
    //  work directory was left by a service that died or was asked to retire in the middle of a push. Waiting out a lease
    //  would leave its work stranded for the lease's length; each such death costs the item one try.
    let _requeued = requeue_claimed(work_dir.as_path(), MAX_TRIES);
    drain_pending(git_dir, handlers, now, None);
}

/// What:
///  Runs the records the releases before one service per worktree left in the common directory of a bare repository, from the
///  instance of a linked worktree: nothing for a repository with a main worktree, for a main worktree and for a directory with
///  no `commondir`.
/// Why:
///   Those releases kept every worktree's records in the common directory. A repository with a main worktree has an instance
///  whose own directory that is, and it runs them (the main worktree's current records live there too), so a linked instance
///  must not. A repository whose worktrees are all linked has no such instance, and its pushes would sit there with no notice.
///   Several linked instances may drain the directory at once, so a claim is taken back only after its lease, which the holder
///  renews while it works.
///
/// In TS you'd write (pseudocode):
/// ```ts
/// await drainCommonRecords(linkedGitDir, handlers, now)
/// ```
pub fn drain_common_records(git_dir: &Path, handlers: &dyn WorkHandlers, now: SystemTime) {
    drain_common_records_with_lease(git_dir, handlers, now, SHARED_CLAIM_LEASE);
}

/// `drain_common_records` with the lease given, so a test can use a short one.
pub fn drain_common_records_with_lease(git_dir: &Path, handlers: &dyn WorkHandlers, now: SystemTime, lease: std::time::Duration) {
    let Some(common) = common_directory_of(git_dir) else { return };
    if common == git_dir || !is_bare_common_directory(common.as_path()) || !work_directory(common.as_path()).is_dir() {
        return;
    }
    let _requeued = requeue_expired(work_directory(common.as_path()).as_path(), now, lease, MAX_TRIES);
    drain_pending(common.as_path(), handlers, now, Some(lease));
}

/// The common directory a linked worktree's Git directory names in its `commondir` file, or nothing for any other directory.
fn common_directory_of(git_dir: &Path) -> Option<PathBuf> {
    let text: String = std::fs::read_to_string(git_dir.join("commondir")).ok()?;
    let named: &Path = Path::new(text.trim_end_matches(['\n', '\r']));
    if named.as_os_str().is_empty() {
        return None;
    }
    // Resolved by the text alone, so the result spells the path the way the worktree's own Git directory does, which is how the
    //  records name their directory.
    let mut resolved: PathBuf = PathBuf::new();
    for part in git_dir.join(named).components() {
        match part {
            std::path::Component::ParentDir => {
                if !resolved.pop() {
                    return None;
                }
            }
            std::path::Component::CurDir => {}
            other => resolved.push(other.as_os_str()),
        }
    }
    return Some(resolved);
}

/// Claims and runs every pending item of one work directory until none is left; `lease` is given when other instances drain
/// the directory too, and the claims held are then renewed so that none of them looks abandoned while it waits or runs.
fn drain_pending(git_dir: &Path, handlers: &dyn WorkHandlers, now: SystemTime, lease: Option<std::time::Duration>) {
    let work_dir: PathBuf = work_directory(git_dir);
    let held: HeldClaims = HeldClaims::default();
    let _renewal: Option<ClaimRenewal> = lease.map(|shared_lease| { return ClaimRenewal::start(work_dir.clone(), held.clone(), shared_lease / RENEWALS_PER_LEASE); });
    loop {
        let mut claimed: Vec<super::work_item::Claimed> = Vec::new();
        loop {
            match claim_next_of(work_dir.as_path(), now, &SERVICE_KINDS) {
                Ok(Some(item)) => {
                    held.hold(item.id.as_str());
                    if lease.is_some() {
                        // The claim carries the time the round started, which a long round makes most of a lease old; the
                        //  stamp is brought up to date now instead of at the next renewal.
                        let _stamped = renew_claim(work_dir.as_path(), item.id.as_str(), SystemTime::now());
                    }
                    claimed.push(item);
                }
                Ok(None) => break,
                Err(reason) => {
                    eprintln!("cli-git-service: {reason}");
                    break;
                }
            }
        }
        if claimed.is_empty() {
            return;
        }
        let (acceptable, refused): (Vec<_>, Vec<_>) = claimed.into_iter().partition(|item| { return is_acceptable(&item.item, git_dir); });
        settle_all(work_dir.as_path(), &held, &refused, FAILED);
        let (pushes, rest): (Vec<_>, Vec<_>) = acceptable.into_iter().partition(|item| { return item.item.kind == WorkKind::Push; });
        let (reviews, upkeep): (Vec<_>, Vec<_>) = rest.into_iter().partition(|item| { return item.item.kind == WorkKind::Review; });
        let mut groups: Vec<((String, String), Vec<super::work_item::Claimed>)> = Vec::new();
        for item in pushes {
            let key = (item.item.repository.clone(), item.item.branch_ref.clone());
            match groups.iter_mut().find(|(existing, _)| { return *existing == key; }) {
                Some((_, members)) => members.push(item),
                None => groups.push((key, vec![item])),
            }
        }
        for (_, members) in groups {
            let items: Vec<WorkItem> = members.iter().map(|member| { return member.item.clone(); }).collect();
            match handlers.push(items.as_slice()) {
                Verdict::Done => {
                    // A push of the branch went through, so an earlier failure to push it is no longer true. The records go
                    //  before the items settle: a service that dies between the two pushes again, and nothing is lost.
                    if let Some(newest) = members.last() {
                        clear_push_failures(git_dir, newest.item.branch_ref.as_str());
                    }
                    settle_all(work_dir.as_path(), &held, &members, DONE);
                }
                Verdict::Failed(text) => {
                    // The record goes before the items settle, so an item in `failed` always has its record.
                    if let Some(newest) = members.last()
                        && let Err(reason) = record_push_failure(git_dir, newest.item.branch_ref.as_str(), newest.item.oid.as_str(), text.as_str())
                    {
                        eprintln!("cli-git-service: the failed push of {} could not be recorded for the person: {reason}", newest.item.oid);
                    }
                    settle_all(work_dir.as_path(), &held, &members, FAILED);
                }
            }
        }
        let mut upkept: Vec<String> = Vec::new();
        for item in upkeep {
            if !upkept.contains(&item.item.repository) {
                upkept.push(item.item.repository.clone());
                if let Verdict::Failed(reason) = handlers.maintenance(&item.item) {
                    eprintln!("cli-git-service: automatic maintenance did not run: {reason}");
                }
            }
            settle_all(work_dir.as_path(), &held, std::slice::from_ref(&item), DONE);
        }
        // Reviews go last: a push is the person's backup, a review is a shadow record, and each is bounded by its budget.
        for item in reviews {
            match handlers.review(&item.item) {
                Verdict::Done => settle_all(work_dir.as_path(), &held, std::slice::from_ref(&item), DONE),
                Verdict::Failed(reason) => {
                    eprintln!("cli-git-service: the review of {} failed: {reason}", item.item.oid);
                    settle_all(work_dir.as_path(), &held, std::slice::from_ref(&item), FAILED);
                }
            }
        }
    }
}

/// How long a claim in the common directory of a bare repository goes without a renewal before another linked worktree's
/// instance runs it again; the holder renews it three times in this span while it works (`service_claim_renewal.rs`).
pub const SHARED_CLAIM_LEASE: std::time::Duration = std::time::Duration::from_secs(900);

/// What:
///  Runs one round of wakes: keeps each wake's review credential in memory, drains each named repository once, and
///  forgets the credentials when the repository is done.
/// Why:
///   The credential is held only for the reviews of the round that brought it; a service that restarts before the round
///  runs has none, and its reviews record the plain "no key" outcome.
///
/// In TS you'd write (pseudocode):
/// ```ts
/// for (const [dir, key] of wakes) { remember(dir, key); await drain(dir); forget(dir) }
/// ```
pub fn run_round(wakes: Vec<Wake>, handlers: &GitHandlers, now: SystemTime) {
    let mut directories: Vec<PathBuf> = Vec::new();
    for (directory, review_key) in wakes {
        if let Some(key) = review_key {
            handlers.remember_review_key(directory.as_path(), key);
        }
        if !directories.contains(&directory) {
            directories.push(directory);
        }
    }
    for directory in directories {
        drain(directory.as_path(), handlers, now);
        drain_common_records(directory.as_path(), handlers, now);
        handlers.forget_review_key(directory.as_path());
    }
}

/// The handlers that run real Git.
pub struct GitHandlers {
    /// The context every Git child of the service runs in.
    pub context: GitContext,
    /// The review credential of each repository whose wake message carried one. Held in memory only, removed when the
    /// round that used it ends.
    review_keys: std::sync::Mutex<std::collections::HashMap<PathBuf, ReviewKey>>,
    /// Where tests point the review's gateway call, so no test reaches the real endpoint.
    #[cfg(test)]
    review_endpoint: Option<String>,
}

/// Methods over the real handlers.
impl GitHandlers {
    /// Handlers that run real Git in `context`, with no review credential yet.
    pub fn new(context: GitContext) -> GitHandlers {
        return GitHandlers {
            context,
            review_keys: std::sync::Mutex::new(std::collections::HashMap::new()),
            #[cfg(test)]
            review_endpoint: None,
        };
    }

    /// The same handlers with the review's gateway call sent to `endpoint`; tests only.
    #[cfg(test)]
    pub fn with_review_endpoint(mut self, endpoint: &str) -> GitHandlers {
        self.review_endpoint = Some(endpoint.to_string());
        return self;
    }

    /// Keep the credential for the reviews of this repository until `forget_review_key`.
    pub fn remember_review_key(&self, git_dir: &Path, key: ReviewKey) {
        self.review_keys.lock().unwrap_or_else(std::sync::PoisonError::into_inner).insert(git_dir.to_path_buf(), key);
    }

    /// Drop the credential of this repository.
    pub fn forget_review_key(&self, git_dir: &Path) {
        self.review_keys.lock().unwrap_or_else(std::sync::PoisonError::into_inner).remove(git_dir);
    }

    fn review_key_of(&self, git_dir: &Path) -> Option<ReviewKey> {
        return self.review_keys.lock().unwrap_or_else(std::sync::PoisonError::into_inner).get(git_dir).cloned();
    }

    fn push_git(&self, item: &WorkItem) -> PushGit {
        let cwd: PathBuf = PathBuf::from(item.repository.as_str());
        return PushGit {
            real_git: self.context.real_git.clone(),
            lfs: read_config_snapshot(&self.context, cwd.as_path()).as_ref().is_none_or(uses_lfs),
            cwd,
            environment: std::env::vars_os().collect(),
        };
    }
}

/// Trait implementation for the real Git.
impl WorkHandlers for GitHandlers {
    fn push(&self, items: &[WorkItem]) -> Verdict {
        let Some(newest) = items.last() else { return Verdict::Done };
        let git: PushGit = self.push_git(newest);
        let mut output: Vec<u8> = Vec::new();
        return match push_once(&git, newest.branch_ref.as_str(), newest.oid.as_str(), &mut output) {
            Ok(result) if result.exit_code == 0 => Verdict::Done,
            Ok(result) => Verdict::Failed(result.shown),
            Err(error) => Verdict::Failed(format!("{error}")),
        };
    }

    fn maintenance(&self, item: &WorkItem) -> Verdict {
        return match run_auto_maintenance_with(&self.context, Path::new(item.repository.as_str()), None) {
            Ok(AutoMaintenanceOutcome::Exited(code)) if code != 0 => Verdict::Failed(format!("it exited with status {code}")),
            Ok(_) => Verdict::Done,
            Err(error) => Verdict::Failed(format!("{error:?}")),
        };
    }

    fn review(&self, item: &WorkItem) -> Verdict {
        let key: Option<ReviewKey> = self.review_key_of(Path::new(item.git_dir.as_str()));
        #[cfg_attr(not(test), expect(unused_mut, reason = "only a test replaces the endpoint"))]
        let mut options: ReviewOptions = options_for(item, &review_variables(key.as_ref()));
        #[cfg(test)]
        if let Some(endpoint) = self.review_endpoint.as_ref() {
            options.endpoint = endpoint.clone();
        }
        return review_item(&self.context, item, options, BUDGET);
    }
}

/// Work consumer controls stay out of the release executable.
#[cfg(test)]
#[path = "service_work_tests.rs"]
mod tests;