git_policy_cli/
index_lock_wait.rs

1//! What:
2//!  Waiting for a foreign `index.lock` under the classification rules:
3//!  a proven-alive owner
4//!       is waited for up to this process's wait limit (`CLI_GIT_LOCK_WAIT_SECONDS`, 30 minutes by default),
5//!  a dead or evidence-free owner is retried on Git's backoff
6//!       schedule until the unproven budget runs out.
7//! Why:
8//!  An index writer must not run beside a landing,
9//!  and a lock nobody can prove alive must
10//!      not block forever.
11//!  Nothing here deletes a lock;
12//!  the caller decides what the failure means
13//!      (`src/index-lock/index-lock-wait.ts`).
14//!
15//! In TS you'd write (pseudocode):
16//! ```ts
17//! await waitForIndexLock({ realIndexPath, timeoutMs, consequence, attempt });
18//! ```
19
20/// The backstop every wait on a live process is bounded by.
21use super::live_lock::{LIVE_WAIT_VARIABLE, live_wait_limit};
22/// Foreign-lock evidence and its verdicts.
23use super::index_lock_evidence::{
24    Evidence, Verdict, classify_index_lock, describe_index_lock_evidence, proven_holder_line,
25};
26
27/// Git's `BACKOFF_MAX_MULTIPLIER`:
28///  the backoff stops growing here.
29pub const BACKOFF_MAX_MULTIPLIER: u64 = 1_000;
30
31/// Git's jitter floor in thousandths of the backoff.
32const JITTER_FLOOR_PERMILLE: u64 = 750;
33
34/// Git's jitter span in thousandths of the backoff.
35const JITTER_SPAN_PERMILLE: u64 = 500;
36
37/// Thousandths per whole.
38const PERMILLE: u64 = 1_000;
39
40/// Longest single poll while a PID file proves the owner alive,
41///  so its release is noticed promptly.
42pub const PROVEN_BY_PID_FILE_POLL_CAP_MS: u64 = 100;
43
44/// Longest single poll while only an open descriptor proves the owner alive;
45///  each poll repeats the
46/// open-holder scan,
47///  so it polls less often.
48pub const PROVEN_BY_DESCRIPTOR_POLL_CAP_MS: u64 = 500;
49
50/// What:
51///  Git's backoff position:
52///  `step` is Git's `n`,
53///  `multiplier` the wait base in milliseconds.
54/// Why:
55///   `BackoffState` in the incumbent.
56///
57/// In TS you'd write (pseudocode):
58/// ```ts
59/// type BackoffState = { step: number; multiplier: number };
60/// ```
61#[derive(Clone, Copy, Debug, Eq, PartialEq)]
62pub struct BackoffState {
63    /// Git's `n`.
64    pub step: u64,
65    /// Git's `multiplier`,
66    ///  the current backoff in milliseconds.
67    pub multiplier: u64,
68}
69
70/// Backoff before the first wait.
71pub const INITIAL_BACKOFF: BackoffState = BackoffState {
72    step: 1,
73    multiplier: 1,
74};
75
76/// What:
77///  One jittered wait from Git's formula `(750 + rand() % 500) * backoff / 1000`,
78///  rounded down.
79/// Why:
80///   Jitter spreads retrying processes apart;
81///  `random` is a value in `[0, 1)`.
82///
83/// In TS you'd write (pseudocode):
84/// ```ts
85/// Math.floor(((750 + Math.floor(random * 500)) * state.multiplier) / 1000);
86/// ```
87pub fn jittered_wait(state: BackoffState, random: f64) -> u64 {
88    let span: u64 = (random * (JITTER_SPAN_PERMILLE as f64)).floor() as u64;
89    return (JITTER_FLOOR_PERMILLE + span) * state.multiplier / PERMILLE;
90}
91
92/// What:
93///  The next backoff:
94///  `multiplier += 2n + 1`,
95///  capped at `BACKOFF_MAX_MULTIPLIER`.
96/// Why:
97///   Adding `2n + 1` makes the multiplier the square of the step,
98///  which is Git's schedule.
99///
100/// In TS you'd write (pseudocode):
101/// ```ts
102/// nextBackoff({ step: 1, multiplier: 1 }); // { step: 2, multiplier: 4 }
103/// ```
104pub fn next_backoff(state: BackoffState) -> BackoffState {
105    let multiplier: u64 = state.multiplier + 2 * state.step + 1;
106    if multiplier > BACKOFF_MAX_MULTIPLIER {
107        return BackoffState {
108            step: state.step,
109            multiplier: BACKOFF_MAX_MULTIPLIER,
110        };
111    }
112    return BackoffState {
113        step: state.step + 1,
114        multiplier,
115    };
116}
117
118/// What:
119///  What one attempt found,
120///  or why the wait ended without a result.
121/// Why:
122///   The caller reports the owner and the consequence;
123///  the attempt itself is the lock check.
124///
125/// In TS you'd write (pseudocode):
126/// ```ts
127/// type IndexLockWaitError = { kind: 'unproven-owner'; evidence; verdict; timeoutMs; consequence } | { kind: 'read'; reason };
128/// ```
129#[derive(Clone, Debug, PartialEq)]
130pub enum WaitFailure {
131    /// A dead or unproven owner outlasted the unproven budget.
132    UnprovenOwner {
133        /// Last attempt's evidence.
134        evidence: Box<Evidence>,
135        /// Last verdict,
136        ///  which is never proven alive.
137        verdict: Verdict,
138        /// Budget that ran out,
139        ///  in milliseconds.
140        timeout_ms: u64,
141        /// What cli-git therefore did not do.
142        consequence: String,
143    },
144    /// A proven-alive owner kept the lock until the wait limit ran out.
145    LiveHolderTimedOut {
146        /// The lock that stayed in place.
147        lock_path: std::path::PathBuf,
148        /// The owner's process ID.
149        pid: i64,
150        /// The owner's command name when readable.
151        command: Option<String>,
152        /// Milliseconds spent waiting for a proven-alive owner.
153        waited_ms: u64,
154        /// The limit that ran out,
155        ///  in milliseconds.
156        limit_ms: u64,
157        /// What cli-git therefore did not do.
158        consequence: String,
159    },
160    /// The lock or its evidence could not be read.
161    Read(String),
162}
163
164/// What:
165///  The side effects of a wait,
166///  injected so tests can drive a fake clock and fake holders.
167/// Why:
168///   The host implementation reads `/proc`,
169///  sleeps and writes standard error;
170///       tests script each step.
171///
172/// In TS you'd write (pseudocode):
173/// ```ts
174/// type IndexLockWaitEffects = { gather; sleep; random; report; now };
175/// ```
176pub trait WaitEffects {
177    /// Whether the lock currently exists (the attempt).
178    fn lock_present(&mut self) -> Result<bool, String>;
179    /// This attempt's evidence,
180    ///  or nothing when the lock was released meanwhile.
181    fn gather(&mut self) -> Result<Option<Evidence>, String>;
182    /// Sleeps for `ms` milliseconds.
183    fn sleep(&mut self, ms: u64);
184    /// A uniform value in `[0, 1)`.
185    fn random(&mut self) -> f64;
186    /// The current time in milliseconds.
187    fn now_ms(&mut self) -> f64;
188    /// Writes one line of diagnostics to standard error.
189    fn report(&mut self, line: &str);
190}
191
192/// What:
193///  The per-wait loop state:
194///  backoff,
195///  time spent unproven,
196///  and whether the holder line was written.
197/// Why:
198///   `state` in `waitForIndexLock`.
199///
200/// In TS you'd write (pseudocode):
201/// ```ts
202/// const state = { backoff: INITIAL_BACKOFF, unprovenMs: 0, reported: false };
203/// ```
204#[derive(Clone, Copy, Debug)]
205struct LoopState {
206    /// Backoff position.
207    backoff: BackoffState,
208    /// Milliseconds spent without a proven owner.
209    unproven_ms: f64,
210    /// Milliseconds spent waiting for a proven-alive owner.
211    live_ms: f64,
212    /// Whether the holder line was written.
213    reported: bool,
214}
215
216/// What:
217///  Wait until the lock is gone,
218///  or fail once a dead or unproven owner outlasts `timeout_ms`
219///  or a live owner outlasts this process's wait limit.
220/// Why:
221///   A live holder gets one diagnostic and the process's wait limit;
222///  other holders get the budget,
223///       which includes the evidence gathering because a `/proc` scan is not free.
224///
225/// In TS you'd write (pseudocode):
226/// ```ts
227/// await waitForIndexLock({ realIndexPath, timeoutMs, consequence, attempt, effects });
228/// ```
229pub fn wait_for_index_lock(
230    effects: &mut dyn WaitEffects,
231    timeout_ms: u64,
232    consequence: &str,
233) -> Result<(), WaitFailure> {
234    let limit_ms: u64 = u64::try_from(live_wait_limit().as_millis()).unwrap_or(u64::MAX);
235    return wait_for_index_lock_within(effects, timeout_ms, limit_ms, consequence);
236}
237
238/// What:
239///  The wait with an explicit limit on the time spent behind a proven-alive owner.
240/// Why:
241///   The default wait and the tests share one loop; only the limit differs.
242///
243/// In TS you'd write (pseudocode):
244/// ```ts
245/// await waitForIndexLock({ realIndexPath, timeoutMs, liveLimitMs, consequence, attempt, effects });
246/// ```
247pub fn wait_for_index_lock_within(
248    effects: &mut dyn WaitEffects,
249    timeout_ms: u64,
250    live_limit_ms: u64,
251    consequence: &str,
252) -> Result<(), WaitFailure> {
253    // `mut` lets each iteration advance the loop state.
254    let mut state: LoopState = LoopState {
255        backoff: INITIAL_BACKOFF,
256        unproven_ms: 0.0,
257        live_ms: 0.0,
258        reported: false,
259    };
260    loop {
261        let iteration_started: f64 = effects.now_ms();
262        let present: bool = match effects.lock_present() {
263            Ok(found) => found,
264            Err(reason) => return Err(WaitFailure::Read(reason)),
265        };
266        if !present {
267            return Ok(());
268        }
269        let evidence: Evidence = match effects.gather() {
270            Ok(Some(found)) => found,
271            // The lock went away between the attempt and the evidence read.
272            Ok(None) => continue,
273            Err(reason) => return Err(WaitFailure::Read(reason)),
274        };
275        let verdict: Verdict = classify_index_lock(&evidence);
276        let random: f64 = effects.random();
277        let backoff_ms: u64 = jittered_wait(state.backoff, random);
278        state.backoff = next_backoff(state.backoff);
279        if let Verdict::ProvenAlive {
280            pid,
281            by_descriptor,
282            command,
283        } = &verdict
284        {
285            if !state.reported {
286                let line: String = proven_holder_line(
287                    *pid,
288                    *by_descriptor,
289                    command.as_deref(),
290                    evidence.lock_path.as_path(),
291                );
292                effects.report(line.as_str());
293                state.reported = true;
294            }
295            let cap: u64 = if *by_descriptor {
296                PROVEN_BY_DESCRIPTOR_POLL_CAP_MS
297            } else {
298                PROVEN_BY_PID_FILE_POLL_CAP_MS
299            };
300            effects.sleep(backoff_ms.min(cap));
301            state.live_ms += effects.now_ms() - iteration_started;
302            if state.live_ms >= live_limit_ms as f64 {
303                return Err(WaitFailure::LiveHolderTimedOut {
304                    lock_path: evidence.lock_path.clone(),
305                    pid: *pid,
306                    command: command.clone(),
307                    waited_ms: state.live_ms as u64,
308                    limit_ms: live_limit_ms,
309                    consequence: String::from(consequence),
310                });
311            }
312            continue;
313        }
314        state.unproven_ms += effects.now_ms() - iteration_started;
315        if state.unproven_ms >= timeout_ms as f64 {
316            return Err(WaitFailure::UnprovenOwner {
317                evidence: Box::new(evidence),
318                verdict,
319                timeout_ms,
320                consequence: String::from(consequence),
321            });
322        }
323        let sleep_started: f64 = effects.now_ms();
324        effects.sleep(backoff_ms);
325        state.unproven_ms += effects.now_ms() - sleep_started;
326    }
327}
328
329/// What:
330///  The diagnostic text of an unproven-owner failure.
331/// Why:
332///   The person needs the lock path,
333///  the owner state,
334///  the budget,
335///  what was not done,
336///  and the evidence.
337///
338/// In TS you'd write (pseudocode):
339/// ```ts
340/// new IndexLockUnprovenOwnerError({ evidence, verdict, timeoutMs, consequence }).message
341/// ```
342pub fn unproven_owner_message(
343    evidence: &Evidence,
344    verdict: &Verdict,
345    timeout_ms: u64,
346    consequence: &str,
347) -> String {
348    let owner: String = match verdict {
349        Verdict::Dead(pid) => format!("whose owner PID {pid} no longer runs"),
350        _ => String::from("whose owner cli-git could not prove alive"),
351    };
352    return format!(
353        "Another process left {}, {owner}; after {timeout_ms} ms (indexLock.unprovenOwnerTimeoutMs) the lock was still there. cli-git left the lock in place and {consequence}. Evidence: {}. Remove the lock only after confirming no Git process is using this repository.",
354        evidence.lock_path.display(),
355        describe_index_lock_evidence(evidence),
356    );
357}
358
359/// What:
360///  The diagnostic text of a live-owner timeout.
361/// Why:
362///   The person needs the lock path, the owner to look at, the limit and its setting,
363///  and what was not done.
364///
365/// In TS you'd write (pseudocode):
366/// ```ts
367/// new IndexLockLiveOwnerTimeoutError({ lockPath, pid, command, waitedMs, limitMs, consequence }).message
368/// ```
369pub fn live_holder_timeout_message(
370    lock_path: &std::path::Path,
371    pid: i64,
372    command: Option<&str>,
373    waited_ms: u64,
374    consequence: &str,
375) -> String {
376    let owner: String = match command {
377        Some(name) => format!("PID {pid} ({name})"),
378        None => format!("PID {pid}"),
379    };
380    return format!(
381        "Gave up after waiting {} seconds for {owner}, which still holds {}; {LIVE_WAIT_VARIABLE} sets how long cli-git waits. cli-git left the lock in place and {consequence}. Look at that process, or raise the limit.",
382        waited_ms / 1_000,
383        lock_path.display(),
384    );
385}
386
387/// Wait controls stay out of the release executable.
388#[cfg(test)]
389#[path = "index_lock_wait_tests.rs"]
390mod tests;
git_policy_cli/index_lock_wait.rs

git_policy_cli/
index_lock_wait.rs
