1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
//! What:
//! How a commit headed for the service notices that the content it prepared was removed from the repository before it
//! landed: the policy read that fails on a missing object, the check of the request's objects after the service could not
//! land it, and the numbered phase markers a test uses to remove content at a chosen instant.
//! Why:
//! A commit's blobs, trees and commit object are unreferenced until the branch moves, and `git gc --prune=now` deletes every
//! unreferenced object at once. The Git manual for `git gc` (the installed 2.56.0 manual and
//! <https://git-scm.com/docs/git-gc>, section NOTES, checked 2026-11-01) says: "when git gc runs concurrently with another
//! process, there is a risk of it deleting an object that the other process is using but hasn't created a reference to.
//! This may just cause the other process to fail or may corrupt the repository if the other process later adds a
//! reference to the deleted object", and for the option itself: "--prune=now prunes loose objects regardless of their age
//! and increases the risk of corruption if another process is writing to the repository concurrently". Git's own
//! mitigation is the two-week grace of the default `--prune` date, which `--prune=now` removes. This commit path therefore
//! says plainly what happened and what to do (`ClientFailure::ContentPruned`), instead of showing a policy event for a
//! machine or a failure of the service. It does not prepare the commit again: the bytes the command captured are what is
//! gone, and reading the worktree again would commit whatever it holds by then.
//!
//! In TS you'd write (pseudocode):
//! ```ts
//! if (contentWasRemoved(events) || preparedContentMissing(request)) fail(CONTENT_PRUNED);
//! ```
use super::candidate_reader::OBJECT_MISSING_PREFIX;
use super::diagnostic_log::warn;
use super::diagnostics::EngineFailureCode;
use super::policy_events::PolicyEvent;
use super::repo_read_layout::ObjectFormat;
use super::repo_read_object::{ObjectKind, from_hex, name_length};
use super::repo_read_odb::ObjectDatabase;
use super::repo_read_tree::{MODE_GITLINK, is_tree_mode, parse_tree};
use super::service_protocol::CommitRequest;
use super::transaction_phase::{Phase, PhaseMarkerError, reach_transaction_phase};
use std::collections::HashSet;
use std::path::Path;
use std::sync::atomic::{AtomicU64, Ordering};
/// Tag used for diagnostics from this module.
const TAG: &str = "cli-git-service";
/// Counts how often this process reached `ServiceStaged`, which numbers the marker files of a commit prepared again.
static STAGED_REACHED: AtomicU64 = AtomicU64::new(0);
/// Counts how often this process reached `ServicePrepared`.
static PREPARED_REACHED: AtomicU64 = AtomicU64::new(0);
/// What:
/// Whether a policy run stopped because an object it had to read is missing from the repository.
/// Why:
/// That is the instant a pruning `git gc` is seen from the policies; preparing the commit again writes the content anew.
/// Other content failures (a Git that did not start, an object of the wrong kind) are not this and keep their event.
///
/// In TS you'd write (pseudocode):
/// ```ts
/// events.some((event) => event.type === 'engine-failure' && event.message.startsWith(OBJECT_MISSING_PREFIX))
/// ```
pub fn content_was_removed(events: &[PolicyEvent]) -> bool {
return events.iter().any(|event| {
return matches!(
event,
PolicyEvent::EngineFailure { code: EngineFailureCode::ContentUnavailable, message, .. } if message.starts_with(OBJECT_MISSING_PREFIX)
);
});
}
/// How many tree entries (and trees) the walk of one batch examines before it stops asking and leaves the rest to Git.
pub const CLOSURE_ENTRY_BUDGET: usize = 5_000_000;
/// How the walk over the trees of a batch ended.
///
/// In TS you'd write (pseudocode):
/// ```ts
/// type ClosureEnd = 'intact' | 'missing' | 'out-of-budget';
/// ```
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum ClosureEnd {
/// Every tree and blob below every root is present, or is left to Git as unreadable.
Intact,
/// An object below a root is gone from the object store.
Missing,
/// The budget ran out first, so nothing is known about the rest.
OutOfBudget,
}
/// What a walk found and what it cost.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
struct ClosureWalk {
/// How it ended.
end: ClosureEnd,
/// Entries and trees examined, which is what the budget counts.
examined: usize,
}
/// What:
/// Whether a tree, a tree below it or a blob in any of them is gone from the object store, for every root of a batch.
/// Why:
/// An amendment carries a whole tree and no list of blobs, and a prune can remove a blob from under a tree that survives
/// until the branch moves. One work stack and one set of examined objects serve the whole batch, so a tree that several
/// requests share (the usual amendment batch changes few files of one large tree) is walked once, and a blob that several
/// trees list is looked up once: the cost is linear in the batch's distinct objects, with no recursion. A found loss ends
/// the whole walk, so a shared set never hides one. The budget counts every tree and entry examined, not only distinct
/// trees; when it runs out the answer is "not known", as for an object that cannot be read here, because holding the
/// landing for an unbounded walk delays every later commit of the worktree. A submodule entry names a commit of another
/// repository and is skipped, and an object that is present but cannot be read here (unreadable, beyond the reader's size
/// limit) is left to Git.
///
/// In TS you'd write (pseudocode):
/// ```ts
/// walkTrees(roots, { shared: true, budget }).some((entry) => !objectStore.has(entry.id))
/// ```
fn walk_tree_closures(database: &ObjectDatabase, roots: &[&str], format: ObjectFormat, budget: usize) -> ClosureWalk {
let mut pending: Vec<Vec<u8>> = roots.iter().filter_map(|root| { return from_hex(root.as_bytes(), format); }).collect();
let mut visited: HashSet<Vec<u8>> = HashSet::new();
let mut examined: usize = 0;
while let Some(name) = pending.pop() {
if !visited.insert(name.clone()) {
continue;
}
if examined >= budget {
return ClosureWalk { end: ClosureEnd::OutOfBudget, examined };
}
examined += 1;
let Some(object) = database.read(name.as_slice()) else {
if database.contains(name.as_slice()) {
continue;
}
return ClosureWalk { end: ClosureEnd::Missing, examined };
};
if object.kind != ObjectKind::Tree {
continue;
}
let Some(entries) = parse_tree(object.bytes.as_slice(), name_length(format)) else { continue };
if entries.len() > budget - examined {
return ClosureWalk { end: ClosureEnd::OutOfBudget, examined };
}
examined += entries.len();
for entry in entries {
if entry.mode == MODE_GITLINK {
continue;
}
if is_tree_mode(entry.mode) {
pending.push(entry.id);
} else if visited.insert(entry.id.clone()) && !database.contains(entry.id.as_slice()) {
return ClosureWalk { end: ClosureEnd::Missing, examined };
}
}
}
return ClosureWalk { end: ClosureEnd::Intact, examined };
}
/// Whether the walk found a loss; an exhausted budget is logged and counts as "not known", which is `false`.
fn tree_closures_missing(database: &ObjectDatabase, roots: &[&str], format: ObjectFormat) -> bool {
let walk: ClosureWalk = walk_tree_closures(database, roots, format, CLOSURE_ENTRY_BUDGET);
if walk.end == ClosureEnd::OutOfBudget {
warn(TAG, &format!("the trees of the batch hold more than {CLOSURE_ENTRY_BUDGET} entries; their objects are not checked here and are left to Git"));
}
return walk.end == ClosureEnd::Missing;
}
/// What:
/// Whether an object the request depends on (a blob it names, the tree of an amend with everything below it, the commit
/// Git built) is gone from the object store the client wrote it to.
/// Why:
/// When the service cannot land a request it answers with a general failure; whether the cause is removed content is
/// something only the objects can say, and reading them here starts no Git. A store the in-process reader cannot open
/// answers `false`, which leaves the general failure as it is.
///
/// In TS you'd write (pseudocode):
/// ```ts
/// namesOf(request).some((name) => !objectStore.has(name))
/// ```
pub fn prepared_content_missing(common_dir: &Path, objects_dir: &Path, object_format: &str, request: &CommitRequest) -> bool {
let format: ObjectFormat = if object_format == ObjectFormat::Sha256.name() { ObjectFormat::Sha256 } else { ObjectFormat::Sha1 };
let Some(database) = ObjectDatabase::open(common_dir, Some(objects_dir), format) else { return false };
let named = request
.changes
.iter()
.filter_map(|change| { return change.blob.as_deref(); })
.chain(request.tree.as_deref())
.chain(request.commit.as_deref());
for name in named {
match from_hex(name.as_bytes(), format) {
Some(bytes) if !database.contains(bytes.as_slice()) => return true,
Some(_) | None => {}
}
}
return request.tree.as_deref().is_some_and(|tree| { return tree_closures_missing(&database, &[tree], format); });
}
/// What:
/// Whether any object the requests of a batch depend on is gone from the repository's object store.
/// Why:
/// The landing asks this before it imports anything or moves the branch: `fast-import` dies on a missing blob and leaves a
/// crash report in the Git directory, and Git moves a branch to a commit whose objects are gone without looking. Loose
/// objects are found with one lookup each; the packs are opened only when an object is not loose. A store the in-process
/// reader cannot open answers `false`, which leaves the decision to Git, as for the client's own check.
///
/// In TS you'd write (pseudocode):
/// ```ts
/// requests.flatMap(namesOf).some((name) => !loose(name) && !(await packs()).has(name))
/// ```
pub fn batch_content_missing(common_dir: &Path, requests: &[&CommitRequest]) -> bool {
let names: Vec<&str> = requests
.iter()
.flat_map(|request| {
return request.changes.iter().filter_map(|change| { return change.blob.as_deref(); }).chain(request.tree.as_deref()).chain(request.commit.as_deref());
})
.collect();
let objects: std::path::PathBuf = common_dir.join("objects");
let is_loose = |name: &str| -> bool { return name.len() > 2 && objects.join(&name[..2]).join(&name[2..]).is_file(); };
let absent: Vec<&str> = names.into_iter().filter(|name| { return !is_loose(name); }).collect();
// An embedded tree is walked whether or not its own object is loose, so a request with one always needs the reader.
let trees: Vec<&str> = requests.iter().filter_map(|request| { return request.tree.as_deref(); }).collect();
let Some(sample) = absent.first().or(trees.first()) else { return false };
let format: ObjectFormat = if sample.len() == ObjectFormat::Sha256.hex_length() { ObjectFormat::Sha256 } else { ObjectFormat::Sha1 };
let Some(database) = ObjectDatabase::open(common_dir, None, format) else { return false };
if absent.iter().any(|name| { return from_hex(name.as_bytes(), format).is_some_and(|bytes| { return !database.contains(bytes.as_slice()); }); }) {
return true;
}
return tree_closures_missing(&database, &trees, format);
}
/// What:
/// Reaches a service-commit phase marker, numbering the files by how often this process reached the phase.
/// Why:
/// A commit prepared again reaches each phase again, and a test that removes content at the phase must be able to answer
/// each time separately (`<phase>-<n>.reached`, `<phase>-<n>.release`).
///
/// In TS you'd write (pseudocode):
/// ```ts
/// await reachServicePhase('service-staged')
/// ```
pub fn reach_service_phase(phase: Phase) -> Result<(), PhaseMarkerError> {
let counter: &AtomicU64 = if phase == Phase::ServiceStaged { &STAGED_REACHED } else { &PREPARED_REACHED };
let number: u64 = counter.fetch_add(1, Ordering::SeqCst) + 1;
return reach_transaction_phase(phase, Some(number));
}
/// Controls stay out of the release executable.
#[cfg(test)]
#[path = "service_pruned_tests.rs"]
mod tests;