1use std::fs::{self, File, OpenOptions};
16use std::io::{self, Write};
17use std::path::{Path, PathBuf};
18use std::sync::Arc;
19use std::time::{Duration, Instant};
20
21use thiserror::Error;
22
23use crate::config::ConfigSet;
24use crate::environment::Environment;
25
26#[derive(Debug, Clone, Copy, PartialEq, Eq)]
28pub enum LockWait {
29 Immediately,
31 Backoff {
33 budget: Duration,
35 },
36 Indefinitely,
38}
39
40impl LockWait {
41 #[must_use]
43 pub fn files_ref_lock(config: Option<&ConfigSet>) -> Self {
44 Self::from_files_ref_lock_timeout_ms(Self::parse_files_ref_lock_timeout_ms(config))
45 }
46
47 #[must_use]
49 pub fn parse_files_ref_lock_timeout_ms(config: Option<&ConfigSet>) -> i64 {
50 config
51 .and_then(|cfg| cfg.get("core.filesreflocktimeout"))
52 .and_then(|v| v.trim().parse::<i64>().ok())
53 .unwrap_or(100)
54 }
55
56 #[must_use]
57 fn from_files_ref_lock_timeout_ms(timeout_ms: i64) -> Self {
58 match timeout_ms {
59 0 => Self::Immediately,
60 ms if ms < 0 => Self::Indefinitely,
61 ms => Self::Backoff {
62 budget: Duration::from_millis(ms as u64),
63 },
64 }
65 }
66
67 #[must_use]
69 pub fn packed_refs(config: Option<&ConfigSet>) -> Self {
70 Self::from_milliseconds_config(config, "core.packedrefstimeout", 1_000)
71 }
72
73 #[must_use]
75 pub fn reftable_tables_list(config: Option<&ConfigSet>) -> Self {
76 Self::from_milliseconds_config(config, "reftable.locktimeout", 1_000)
77 }
78
79 #[must_use]
81 pub fn files_ref_lock_for_git_dir(git_dir: &Path) -> Self {
82 let config = ConfigSet::load(&Environment::empty(), Some(git_dir), true).ok();
83 Self::files_ref_lock(config.as_ref())
84 }
85
86 #[must_use]
88 pub fn packed_refs_for_git_dir(git_dir: &Path) -> Self {
89 let config = ConfigSet::load(&Environment::empty(), Some(git_dir), true).ok();
90 Self::packed_refs(config.as_ref())
91 }
92
93 fn from_milliseconds_config(config: Option<&ConfigSet>, key: &str, default_ms: u64) -> Self {
94 let ms = config
95 .and_then(|cfg| cfg.get(key))
96 .and_then(|v| v.trim().parse::<i64>().ok())
97 .map(|v| v.max(0) as u64)
98 .unwrap_or(default_ms);
99 if ms == 0 {
100 Self::Immediately
101 } else {
102 Self::Backoff {
103 budget: Duration::from_millis(ms),
104 }
105 }
106 }
107}
108
109#[derive(Debug, Error)]
111#[non_exhaustive]
112pub enum LockError {
113 #[error("lock held at {path}")]
115 Held {
116 path: PathBuf,
118 },
119
120 #[error("lock I/O at {path}")]
122 Io {
123 path: PathBuf,
125 #[source]
127 source: io::Error,
128 },
129}
130
131pub trait LockClock: Send + Sync {
133 fn now(&self) -> Instant;
135 fn sleep(&self, duration: Duration);
137}
138
139pub trait LockJitter: Send + Sync {
141 fn fraction(&self, lock_path: &Path, attempt: u32) -> f64;
143}
144
145#[derive(Debug, Default, Clone, Copy)]
147pub struct SystemLockJitter;
148
149impl LockJitter for SystemLockJitter {
150 fn fraction(&self, _lock_path: &Path, _attempt: u32) -> f64 {
151 let mut buf = [0u8; 8];
152 if getrandom::getrandom(&mut buf).is_ok() {
153 let n = u64::from_le_bytes(buf);
154 (n % 10_000) as f64 / 10_000.0
155 } else {
156 0.5
157 }
158 }
159}
160
161#[derive(Debug, Default, Clone, Copy)]
163pub struct SystemLockClock;
164
165impl LockClock for SystemLockClock {
166 fn now(&self) -> Instant {
167 Instant::now()
168 }
169
170 fn sleep(&self, duration: Duration) {
171 if !duration.is_zero() {
172 std::thread::sleep(duration);
173 }
174 }
175}
176
177pub struct LockFile {
182 lock_path: PathBuf,
183 commit_path: Option<PathBuf>,
184 file: Option<File>,
185 committed: bool,
186 wrote_holder_pid: bool,
187}
188
189impl LockFile {
190 pub fn acquire(target: &Path, wait: LockWait) -> Result<Self, LockError> {
201 Self::acquire_with_clock(
202 target,
203 wait,
204 Arc::new(SystemLockClock),
205 Arc::new(SystemLockJitter),
206 )
207 }
208
209 pub fn acquire_with_clock(
211 target: &Path,
212 wait: LockWait,
213 clock: Arc<dyn LockClock>,
214 jitter: Arc<dyn LockJitter>,
215 ) -> Result<Self, LockError> {
216 let to_lock_error = |source: io::Error| {
217 if crate::atomic_replace::is_lock_acquire_contention(&source, target) {
218 LockError::Held {
219 path: target.to_path_buf(),
220 }
221 } else {
222 LockError::Io {
223 path: target.to_path_buf(),
224 source,
225 }
226 }
227 };
228 let open = || OpenOptions::new().write(true).create_new(true).open(target);
229 let mut parent_ready = false;
232 retry_while_held(
233 target,
234 || wait,
235 clock.as_ref(),
236 jitter.as_ref(),
237 || match open() {
238 Err(_) if !parent_ready => {
239 parent_ready = true;
240 if let Some(parent) = target.parent() {
241 fs::create_dir_all(parent).map_err(|source| LockError::Io {
242 path: target.to_path_buf(),
243 source,
244 })?;
245 }
246 open().map_err(to_lock_error)
247 }
248 other => other.map_err(to_lock_error),
249 },
250 |err| matches!(err, LockError::Held { .. }),
251 )
252 .map(|file| Self {
253 lock_path: target.to_path_buf(),
254 commit_path: None,
255 file: Some(file),
256 committed: false,
257 wrote_holder_pid: false,
258 })
259 }
260
261 #[must_use]
263 pub fn with_commit_target(mut self, path: &Path) -> Self {
264 self.commit_path = Some(path.to_path_buf());
265 self
266 }
267
268 #[must_use]
270 pub fn lock_path(&self) -> &Path {
271 &self.lock_path
272 }
273
274 pub fn as_file(&self) -> Result<&File, LockError> {
281 self.file.as_ref().ok_or_else(|| LockError::Io {
282 path: self.lock_path.clone(),
283 source: io::Error::new(io::ErrorKind::NotConnected, "lock file handle missing"),
284 })
285 }
286
287 pub fn as_file_mut(&mut self) -> Result<&mut File, LockError> {
293 self.file.as_mut().ok_or_else(|| LockError::Io {
294 path: self.lock_path.clone(),
295 source: io::Error::new(io::ErrorKind::NotConnected, "lock file handle missing"),
296 })
297 }
298
299 pub(crate) fn into_open_file(mut self) -> Result<(PathBuf, File), LockError> {
301 self.committed = true;
302 let file = self.file.take().ok_or_else(|| LockError::Io {
303 path: self.lock_path.clone(),
304 source: io::Error::new(io::ErrorKind::NotConnected, "lock file handle missing"),
305 })?;
306 Ok((self.lock_path.clone(), file))
307 }
308
309 pub fn commit(self) -> Result<(), LockError> {
316 let dest = self.commit_path.clone().ok_or_else(|| LockError::Io {
317 path: self.lock_path.clone(),
318 source: io::Error::new(
319 io::ErrorKind::InvalidInput,
320 "lock commit target not configured",
321 ),
322 })?;
323 self.commit_to(&dest)
324 }
325
326 pub fn commit_to(mut self, other_path: &Path) -> Result<(), LockError> {
332 let file = self.file.take().ok_or_else(|| LockError::Io {
333 path: self.lock_path.clone(),
334 source: io::Error::new(io::ErrorKind::NotConnected, "lock file handle missing"),
335 })?;
336 file.sync_all().map_err(|source| LockError::Io {
337 path: self.lock_path.clone(),
338 source,
339 })?;
340 drop(file);
341 crate::atomic_replace::replace_file(&self.lock_path, other_path).map_err(|source| {
342 LockError::Io {
343 path: self.lock_path.clone(),
344 source,
345 }
346 })?;
347 self.remove_holder_pid_sidecar();
348 self.committed = true;
349 Ok(())
350 }
351
352 pub fn write_holder_pid(&mut self) -> Result<(), LockError> {
358 write_lock_pid_file(&pid_path_for_lock(&self.lock_path)).map_err(|source| {
359 LockError::Io {
360 path: self.lock_path.clone(),
361 source,
362 }
363 })?;
364 self.wrote_holder_pid = true;
365 Ok(())
366 }
367
368 pub fn rollback(mut self) {
370 if self.committed {
371 return;
372 }
373 drop(self.file.take());
374 let _ = fs::remove_file(&self.lock_path);
375 self.remove_holder_pid_sidecar();
376 self.committed = true;
377 }
378
379 fn remove_holder_pid_sidecar(&self) {
380 if self.wrote_holder_pid {
381 let _ = fs::remove_file(pid_path_for_lock(&self.lock_path));
382 }
383 }
384
385 pub fn detach(mut self) {
387 self.committed = true;
388 }
389}
390
391impl Write for LockFile {
392 fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
393 match self.file.as_mut() {
394 Some(file) => file.write(buf),
395 None => Err(io::Error::new(
396 io::ErrorKind::NotConnected,
397 "lock file handle missing",
398 )),
399 }
400 }
401
402 fn flush(&mut self) -> io::Result<()> {
403 match self.file.as_mut() {
404 Some(file) => file.flush(),
405 None => Err(io::Error::new(
406 io::ErrorKind::NotConnected,
407 "lock file handle missing",
408 )),
409 }
410 }
411}
412
413impl Drop for LockFile {
414 fn drop(&mut self) {
415 if self.committed {
416 return;
417 }
418 drop(self.file.take());
419 let _ = fs::remove_file(&self.lock_path);
420 self.remove_holder_pid_sidecar();
421 }
422}
423
424pub(crate) fn retry_while_held<T, E>(
437 key: &Path,
438 wait: impl FnOnce() -> LockWait,
439 clock: &dyn LockClock,
440 jitter: &dyn LockJitter,
441 mut attempt: impl FnMut() -> Result<T, E>,
442 is_held: impl Fn(&E) -> bool,
443) -> Result<T, E> {
444 let start = clock.now();
445 let mut err = match attempt() {
446 Err(err) if is_held(&err) => err,
447 other => return other,
448 };
449 let deadline = match wait() {
450 LockWait::Immediately => return Err(err),
451 LockWait::Backoff { budget } => Some(start + budget),
452 LockWait::Indefinitely => None,
453 };
454 let mut tries: u32 = 0;
455 loop {
456 if deadline.is_some_and(|deadline| clock.now() >= deadline) {
457 return Err(err);
458 }
459 let delay = backoff_delay(key, tries, jitter);
460 tries = tries.saturating_add(1);
461 clock.sleep(match deadline {
462 Some(deadline) => delay.min(deadline.saturating_duration_since(clock.now())),
463 None => delay,
464 });
465 match attempt() {
466 Err(held) if is_held(&held) => err = held,
467 other => return other,
468 }
469 }
470}
471
472fn backoff_delay(lock_path: &Path, attempt: u32, jitter: &dyn LockJitter) -> Duration {
473 const INITIAL_MS: u64 = 1;
474 const CAP_MS: u64 = 50;
475 let shift = attempt.min(16);
476 let base_ms = (INITIAL_MS.saturating_mul(1_u64 << shift)).min(CAP_MS);
477 let span_ms = base_ms / 2 + 1;
478 let jitter_ms = (jitter.fraction(lock_path, attempt) * span_ms as f64) as u64;
479 Duration::from_millis(base_ms.saturating_add(jitter_ms))
480}
481
482#[derive(Debug, Clone, Copy, PartialEq, Eq)]
487pub struct LockHolder {
488 pub pid: u64,
490 pub running: bool,
495}
496
497impl LockHolder {
498 #[must_use]
504 pub fn read(lock_path: &Path) -> Option<Self> {
505 let pid = read_lock_pid(&pid_path_for_lock(lock_path))?;
506 Some(Self {
507 pid,
508 running: is_process_running(pid),
509 })
510 }
511
512 pub fn remove_if_stale(lock_path: &Path) -> io::Result<bool> {
523 let Some(holder) = Self::read(lock_path) else {
524 return Ok(false);
525 };
526 if holder.running {
527 return Ok(false);
528 }
529 let pid_path = pid_path_for_lock(lock_path);
530 if lock_path.is_file() && pid_path.is_file() {
531 let lock_mtime = fs::metadata(lock_path)?.modified()?;
532 let pid_mtime = fs::metadata(&pid_path)?.modified()?;
533 if lock_mtime > pid_mtime {
534 return Ok(false);
535 }
536 }
537 for path in [lock_path, &pid_path] {
538 match fs::remove_file(path) {
539 Ok(()) => {}
540 Err(e) if e.kind() == io::ErrorKind::NotFound => {}
541 Err(e) => return Err(e),
542 }
543 }
544 Ok(true)
545 }
546}
547
548#[must_use]
555pub fn grit_lockfile_pid_enabled(config: Option<&ConfigSet>, git_dir: Option<&Path>) -> bool {
556 fn from_config(cfg: &ConfigSet) -> bool {
557 match cfg.get_bool("core.lockfilepid") {
558 Some(Ok(false)) => false,
559 Some(Ok(true)) | Some(Err(_)) | None => true,
560 }
561 }
562 if let Some(cfg) = config {
563 return from_config(cfg);
564 }
565 if let Some(dir) = git_dir {
566 if let Ok(cfg) = ConfigSet::load(&Environment::empty(), Some(dir), true) {
567 return from_config(&cfg);
568 }
569 }
570 true
571}
572
573pub fn pid_path_for_lock(lock_path: &Path) -> PathBuf {
575 let file_name = lock_path
576 .file_name()
577 .map(|s| s.to_string_lossy().into_owned())
578 .unwrap_or_else(|| "index.lock".to_owned());
579 let base = file_name.strip_suffix(".lock").unwrap_or(&file_name);
580 lock_path.with_file_name(format!("{base}~pid.lock"))
581}
582
583pub(crate) fn write_lock_pid_file(pid_path: &Path) -> io::Result<()> {
584 let mut file = OpenOptions::new()
585 .write(true)
586 .create(true)
587 .truncate(true)
588 .open(pid_path)?;
589 writeln!(file, "pid {}", std::process::id())?;
590 Ok(())
591}
592
593fn read_lock_pid(pid_path: &Path) -> Option<u64> {
594 let raw = fs::read_to_string(pid_path).ok()?;
595 let trimmed = raw.trim();
596 trimmed
597 .strip_prefix("pid ")
598 .unwrap_or(trimmed)
599 .trim()
600 .parse::<u64>()
601 .ok()
602}
603
604pub(crate) fn is_process_running(pid: u64) -> bool {
606 if pid == 0 || pid > i32::MAX as u64 {
607 return false;
608 }
609 #[cfg(windows)]
610 {
611 use windows_sys::Win32::Foundation::CloseHandle;
612 use windows_sys::Win32::System::Threading::{
613 OpenProcess, PROCESS_QUERY_LIMITED_INFORMATION,
614 };
615 unsafe {
617 let handle = OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, 0, pid as u32);
618 if handle.is_null() {
619 return false;
620 }
621 CloseHandle(handle);
622 true
623 }
624 }
625 #[cfg(all(unix, target_os = "linux"))]
626 {
627 Path::new(&format!("/proc/{pid}")).exists()
628 }
629 #[cfg(all(unix, not(target_os = "linux")))]
630 {
631 use rustix::io::Errno;
632 use rustix::process::{test_kill_process, Pid};
633 let Some(pid) = Pid::from_raw(pid as i32) else {
634 return false;
635 };
636 match test_kill_process(pid) {
637 Ok(()) => true,
638 Err(Errno::SRCH) => false,
639 Err(Errno::PERM) => true,
640 Err(_) => false,
641 }
642 }
643}
644
645#[cfg(test)]
646mod tests {
647 use super::*;
648 use std::sync::Mutex;
649
650 struct FakeLockClock {
651 now: Mutex<Instant>,
652 sleeps: Mutex<Vec<Duration>>,
653 }
654
655 impl FakeLockClock {
656 fn new(start: Instant) -> Arc<Self> {
657 Arc::new(Self {
658 now: Mutex::new(start),
659 sleeps: Mutex::new(Vec::new()),
660 })
661 }
662
663 fn sleeps(&self) -> Vec<Duration> {
664 self.sleeps.lock().unwrap().clone()
665 }
666 }
667
668 impl LockClock for FakeLockClock {
669 fn now(&self) -> Instant {
670 *self.now.lock().unwrap()
671 }
672
673 fn sleep(&self, duration: Duration) {
674 self.sleeps.lock().unwrap().push(duration);
675 *self.now.lock().unwrap() += duration;
676 }
677 }
678
679 #[test]
680 fn drop_without_commit_removes_own_lock() {
681 let dir = tempfile::tempdir().unwrap();
682 let lock = dir.path().join("resource.lock");
683 {
684 let _lock = LockFile::acquire(&lock, LockWait::Immediately).unwrap();
685 assert!(lock.is_file());
686 }
687 assert!(!lock.exists());
688 }
689
690 #[test]
691 fn commit_to_replaces_target() {
692 let dir = tempfile::tempdir().unwrap();
693 let target = dir.path().join("resource");
694 let lock_path = dir.path().join("resource.lock");
695 let mut lock = LockFile::acquire(&lock_path, LockWait::Immediately).unwrap();
696 lock.write_all(b"payload\n").unwrap();
697 lock.commit_to(&target).unwrap();
698 assert!(!lock_path.exists());
699 assert_eq!(fs::read_to_string(&target).unwrap(), "payload\n");
700 }
701
702 struct FixedJitter(f64);
703
704 impl LockJitter for FixedJitter {
705 fn fraction(&self, _lock_path: &Path, _attempt: u32) -> f64 {
706 self.0
707 }
708 }
709
710 #[test]
711 fn held_when_budget_exhausted() {
712 let dir = tempfile::tempdir().unwrap();
713 let lock_path = dir.path().join("x.lock");
714 fs::write(&lock_path, b"foreign").unwrap();
715 let start = Instant::now();
716 let clock = FakeLockClock::new(start);
717 let err = match LockFile::acquire_with_clock(
718 &lock_path,
719 LockWait::Backoff {
720 budget: Duration::from_millis(10),
721 },
722 clock.clone(),
723 Arc::new(FixedJitter(0.5)),
724 ) {
725 Err(e) => e,
726 Ok(_) => panic!("expected lock held error"),
727 };
728 assert!(matches!(err, LockError::Held { .. }));
729 assert!(lock_path.is_file());
730 assert_eq!(fs::read_to_string(&lock_path).unwrap(), "foreign");
731 assert!(!clock.sleeps().is_empty());
732 }
733
734 #[test]
735 fn pid_sidecar_path_replaces_lock_suffix() {
736 assert_eq!(
737 pid_path_for_lock(Path::new("/r/.git/index.lock")),
738 Path::new("/r/.git/index~pid.lock")
739 );
740 assert_eq!(
741 pid_path_for_lock(Path::new("/r/.git/refs/heads/main.lock")),
742 Path::new("/r/.git/refs/heads/main~pid.lock")
743 );
744 assert_eq!(
745 pid_path_for_lock(Path::new("/r/odd")),
746 Path::new("/r/odd~pid.lock")
747 );
748 }
749
750 #[test]
751 fn remove_if_stale_skips_when_lock_is_newer_than_sidecar() {
752 use std::thread;
753 use std::time::Duration;
754
755 let dir = tempfile::tempdir().unwrap();
756 let lock_path = dir.path().join("index.lock");
757 fs::write(&lock_path, b"").unwrap();
758 let pid_path = pid_path_for_lock(&lock_path);
759 fs::write(&pid_path, "pid 999999999\n").unwrap();
760 thread::sleep(Duration::from_millis(20));
761 fs::write(&lock_path, b"replaced").unwrap();
762
763 assert!(!LockHolder::remove_if_stale(&lock_path).unwrap());
764 assert!(lock_path.is_file());
765 assert!(pid_path.is_file());
766 }
767
768 #[test]
769 fn held_converts_to_typed_error_with_holder() {
770 let dir = tempfile::tempdir().unwrap();
771 let lock_path = dir.path().join("index.lock");
772 fs::write(&lock_path, b"").unwrap();
773 fs::write(
774 dir.path().join("index~pid.lock"),
775 format!("pid {}\n", std::process::id()),
776 )
777 .unwrap();
778 let err = LockFile::acquire(&lock_path, LockWait::Immediately)
779 .err()
780 .expect("held");
781 match crate::error::Error::from(err) {
782 crate::error::Error::LockHeld { path, holder } => {
783 assert_eq!(path, lock_path);
784 assert_eq!(
785 holder,
786 Some(LockHolder {
787 pid: u64::from(std::process::id()),
788 running: true,
789 })
790 );
791 }
792 other => panic!("expected LockHeld, got {other:?}"),
793 }
794 }
795
796 #[test]
797 fn io_failure_converts_to_io_error() {
798 let err = LockError::Io {
799 path: PathBuf::from("x.lock"),
800 source: io::Error::from(io::ErrorKind::PermissionDenied),
801 };
802 assert!(matches!(
803 crate::error::Error::from(err),
804 crate::error::Error::Io(e) if e.kind() == io::ErrorKind::PermissionDenied
805 ));
806 }
807
808 #[test]
809 fn rollback_removes_own_lock() {
810 let dir = tempfile::tempdir().unwrap();
811 let lock = dir.path().join("resource.lock");
812 let lock_file = LockFile::acquire(&lock, LockWait::Immediately).unwrap();
813 assert!(lock.is_file());
814 lock_file.rollback();
815 assert!(!lock.exists());
816 }
817
818 #[test]
819 fn files_ref_lock_timeout_uses_milliseconds() {
820 let mut cfg = ConfigSet::new();
821 cfg.add_command_override("core.filesreflocktimeout", "100")
822 .expect("set timeout");
823 assert_eq!(LockWait::parse_files_ref_lock_timeout_ms(Some(&cfg)), 100);
824 match LockWait::files_ref_lock(Some(&cfg)) {
825 LockWait::Backoff { budget } => {
826 assert_eq!(budget, Duration::from_millis(100));
827 }
828 other => panic!("expected backoff, got {other:?}"),
829 }
830 }
831
832 #[test]
833 fn files_ref_lock_timeout_negative_retries_indefinitely() {
834 let mut cfg = ConfigSet::new();
835 cfg.add_command_override("core.filesreflocktimeout", "-1")
836 .expect("set timeout");
837 assert!(matches!(
838 LockWait::files_ref_lock(Some(&cfg)),
839 LockWait::Indefinitely
840 ));
841 }
842
843 #[test]
844 fn indefinite_wait_succeeds_when_lock_cleared() {
845 let dir = tempfile::tempdir().unwrap();
846 let lock_path = dir.path().join("x.lock");
847 fs::write(&lock_path, b"foreign").unwrap();
848 let clock = FakeLockClock::new(Instant::now());
849 let lock_path_thread = lock_path.clone();
850 std::thread::spawn(move || {
851 std::thread::sleep(Duration::from_millis(5));
852 let _ = fs::remove_file(&lock_path_thread);
853 });
854 let _guard = LockFile::acquire_with_clock(
855 &lock_path,
856 LockWait::Indefinitely,
857 clock,
858 Arc::new(FixedJitter(0.0)),
859 )
860 .expect("indefinite wait should eventually acquire");
861 }
862
863 #[test]
864 fn retry_while_held_resolves_wait_lazily_and_counts_attempt_time() {
865 let key = Path::new("refs");
866 let jitter = FixedJitter(0.0);
867 let clock = FakeLockClock::new(Instant::now());
868
869 let ok: Result<u32, u32> = retry_while_held(
870 key,
871 || panic!("wait resolved without contention"),
872 clock.as_ref(),
873 &jitter,
874 || Ok(7),
875 |_| true,
876 );
877 assert_eq!(ok, Ok(7));
878
879 let mut calls = 0;
880 let other = retry_while_held(
881 key,
882 || LockWait::Indefinitely,
883 clock.as_ref(),
884 &jitter,
885 || {
886 calls += 1;
887 Err::<(), _>("io")
888 },
889 |e| *e == "held",
890 );
891 assert_eq!((other, calls), (Err("io"), 1));
892 assert!(clock.sleeps().is_empty());
893
894 let mut calls = 0;
896 let slow = retry_while_held(
897 key,
898 || LockWait::Backoff {
899 budget: Duration::from_millis(10),
900 },
901 clock.as_ref(),
902 &jitter,
903 || {
904 calls += 1;
905 clock.sleep(Duration::from_millis(10));
906 Err::<(), _>(calls)
907 },
908 |_| true,
909 );
910 assert_eq!(slow, Err(1));
911
912 let mut calls = 0;
913 let later = retry_while_held(
914 key,
915 || LockWait::Backoff {
916 budget: Duration::from_secs(1),
917 },
918 clock.as_ref(),
919 &jitter,
920 || {
921 calls += 1;
922 if calls < 4 {
923 Err(calls)
924 } else {
925 Ok(calls)
926 }
927 },
928 |_| true,
929 );
930 assert_eq!(later, Ok(4));
931 }
932}