gunnar_server/mirror_fetch.rs
1//! **What `Mirror.Sync` actually does**: one fetch from an upstream, into this
2//! box's own storage engine, and one compare-and-swap that moves the
3//! references.
4//!
5//! # There is one client, and two ways to reach a remote
6//!
7//! [`gunnar_wire::fetch_pack`] is the whole protocol — the advertisement, the
8//! `ls-refs`, the want/have negotiation, the sideband demultiplex, the packfile
9//! streamed into a [`PackIngest`](gunnar_wire::PackIngest). It takes a
10//! `&mut dyn BufRead` and a `&mut dyn Write` and cannot tell what is behind
11//! them, which is the seam this module fills twice and only twice:
12//!
13//! | remote | the pair | who built it |
14//! |---|---|---|
15//! | `ssh://`, `git+ssh://`, `git@host:path` | a russh channel's two halves | `nornir-git`, through `gunnar_client::connect_upload_pack` — the **same dialler** replication pushes over |
16//! | `https://`, `http://` | a buffered request/response adapter | [`crate::mirror_http`] |
17//!
18//! Neither is a second protocol implementation, and that is the point. A bug in
19//! negotiation is one bug, in one file, in `gunnar-wire`.
20//!
21//! # This module writes the references, and `fetch_pack` deliberately does not
22//!
23//! `fetch_pack`'s module doc states the rule: it returns a **plan**
24//! (`FetchReport::updates`, each entry a name, the old value and the new one)
25//! and applies nothing, because applying a whole mirror's plan with no gate, no
26//! atomicity across the batch and no way for an operator to see it first is not
27//! something a protocol crate should be able to do.
28//!
29//! So the apply is here, and it is **one [`GitOps::put_refs_cas`]** with the
30//! plan's own `old` as every expectation. That is not tidiness:
31//!
32//! * A mirror is not the only writer. `git push` reaches the same repository
33//! over SSH and over smart HTTP, and a fetch that read a tip, spent four
34//! minutes on the network and then wrote unconditionally would **silently
35//! destroy** whatever landed in between. The `old` in the plan is read from
36//! the same `GitOps::refs()` snapshot the `have` list came from, so the CAS
37//! is checking exactly the value the fetch reasoned about.
38//! * A CAS failure is a **FAILED job that names the reference**, never a skip.
39//! "Synced" about a repository where one branch quietly did not move is the
40//! answer that makes a mirror untrustworthy, and it reads identically to a
41//! mirror that worked.
42//! * Every edit or none. A half-applied plan leaves the repository at a state
43//! no upstream ever had.
44//!
45//! # …and the batch goes through branch protection first
46//!
47//! A mirror moves references, and until 2026-09-11 it was the only writer on
48//! this box that moved them without asking [`BranchProtectionGate`]. That is
49//! not a smaller version of a push: `Mirror.Sync` needs **Write** and
50//! `Policy.SetProtection` needs **Admin**, so a principal who may not configure
51//! a protection rule could register a mirror against the repository it protects
52//! and have the upstream's tips applied over it — a non-fast-forward onto a ref
53//! whose rule says `allow_force_push: false`, applied by the one path that
54//! never looked. See [`Protection`] for what is decided here and what is
55//! refused as undecidable.
56//!
57//! # The credential is never in this module's output
58//!
59//! The value is fetched from the sealed secret store, held for the length of
60//! one dial, and never formatted. An SSH key is materialised into a
61//! `0600` temporary file because `nornir-git` authenticates from a path and
62//! this tree does not fork a second dialler to change that (one implementation); the file is
63//! unlinked when the fetch returns, on every path including a panic, because
64//! [`tempfile::NamedTempFile`] unlinks on `Drop`. A token becomes an
65//! `Authorization` header value inside [`crate::mirror_http`] and nothing else.
66//! Errors that mention a URL go through [`scrub`] first.
67
68use std::io::{BufRead, Write};
69use std::sync::atomic::{AtomicU64, Ordering};
70use std::sync::Arc;
71
72use gix_hash::ObjectId;
73use gunnar_core::MirrorCredentialKind;
74use gunnar_store::git_storage_trait::{GitOps, RefCas};
75use gunnar_wire::fetch_pack::{self, FetchPack, FetchProgress, FetchReport, LocalRefs};
76
77use crate::job_queue::Reporter;
78use crate::mirror_queue::{Mirror, MirrorOrder};
79use crate::stores::ServerStores;
80
81/// The tracing target a sync's lines carry — the same one every other git
82/// transfer on this box uses, so `RUST_LOG=gunnar::git=info` selects a mirror
83/// as well as a clone.
84const TARGET_GIT: &str = "gunnar::git";
85
86/// **The `known_hosts` a mirror's SSH dial trusts**, under the data root.
87///
88/// Not `~/.ssh/known_hosts`, and this is the same ruling replication's
89/// `REPLICATION_KNOWN_HOSTS` makes for the push direction: a server's identity
90/// is the box, not a home directory. The appliance has no home directory at
91/// all, so the default would be a file that cannot be written and a
92/// trust-on-first-use that silently never remembers — every sync a first
93/// contact, which is the same as no host-key checking with extra steps.
94///
95/// A separate file from replication's, because they are separate rosters: the
96/// standbys this box pushes to and the upstreams it pulls from are different
97/// sets of machines, and a pin acquired for one is not consent for the other.
98pub const MIRROR_KNOWN_HOSTS: &str = "mirror_known_hosts";
99
100/// **The file that makes a mirror's SSH dial refuse an unknown host key**,
101/// under the data root beside [`MIRROR_KNOWN_HOSTS`].
102///
103/// Without it a mirror dials trust-on-first-use: an upstream this box has never
104/// seen is recorded and trusted, and only a *changed* key is refused. That is
105/// the right default for a developer's laptop and a defensible one for a server
106/// being brought up without a provisioning step — it is what replication's own
107/// `strict_host_key` defaults to, and its comment already says that on "is the
108/// right setting for anything that outlives an afternoon". The mirror side had
109/// no way to say on at all: the policy was a literal in [`dial_ssh`].
110///
111/// A file rather than a field on `MirrorRecord`, for the reason
112/// [`MIRROR_KNOWN_HOSTS`] is a file under the data root: the roster of hosts
113/// this box will pull from is the box's, not one repository's, and the pin it
114/// is checked against is already per-box. Create it once, and every mirror on
115/// the box requires a `known_hosts` entry that somebody put there.
116pub const MIRROR_STRICT_HOST_KEY: &str = "mirror_strict_host_key";
117
118/// Does this box refuse an unknown upstream host key? See
119/// [`MIRROR_STRICT_HOST_KEY`].
120///
121/// The presence of the file is read at each dial rather than cached, so an
122/// operator who creates it does not have to restart the server to mean it.
123pub fn strict_host_keys(data_root: &std::path::Path) -> bool {
124 data_root.join(MIRROR_STRICT_HOST_KEY).exists()
125}
126
127/// **The file that lets this box's mirrors dial an address that is not
128/// publicly routable**, under the data root beside [`MIRROR_KNOWN_HOSTS`].
129///
130/// A file and not an environment variable, for the reason `wg.rs` writes out at
131/// length: an env var is process-global, so a rule that can only be exercised
132/// by `set_var` is a rule whose tests flake the moment `cargo test` runs them
133/// on two threads. A file under the data root is per-box, survives a restart,
134/// is visible to an operator looking at the box, and each test gets its own.
135///
136/// It exists because refusing every private destination outright would be a
137/// product regression, not a fix: mirroring from a forge on the operator's own
138/// LAN is an ordinary thing for an appliance to do. What must not be ordinary
139/// is a **low-privileged** caller getting it for free — see
140/// [`check_destination`].
141pub const MIRROR_ALLOW_PRIVATE: &str = "mirror_allow_private_destinations";
142
143/// **The credential a mirror dials with**, in the one shape it exists in
144/// memory.
145///
146/// Deliberately not `Debug`, `Clone` or `Serialize`. The compiler is the only
147/// mechanism that reliably stops a secret being formatted into a log line by
148/// somebody adding a `{:?}` three months from now, so the derive that would
149/// allow it is simply absent.
150pub enum Credential {
151 /// No credential. A public upstream needs none, and a mirror of one works.
152 None,
153 /// An OpenSSH private key, for an `ssh://` upstream.
154 SshKey(Vec<u8>),
155 /// A token, for an `https://` upstream. `username` empty means `Bearer`.
156 Token { username: String, secret: String },
157}
158
159impl Credential {
160 fn kind(&self) -> MirrorCredentialKind {
161 match self {
162 Credential::None => MirrorCredentialKind::None,
163 Credential::SshKey(_) => MirrorCredentialKind::SshKey,
164 Credential::Token { .. } => MirrorCredentialKind::Token,
165 }
166 }
167}
168
169/// **Which transport a remote URL names.**
170///
171/// Decided by the URL and by nothing else — not by which credential happens to
172/// be stored, and not by trying one and falling back. A fallback would turn "I
173/// gave you the wrong kind of key" into "your upstream is down", and those go
174/// to two different people.
175#[derive(Debug, Clone, Copy, PartialEq, Eq)]
176pub enum Wire {
177 /// `ssh://`, `git+ssh://`, or the scp-like `git@host:ns/repo.git`.
178 Ssh,
179 /// `https://` or `http://`.
180 Http,
181}
182
183impl Wire {
184 /// The word an operator sees in a refusal.
185 fn name(self) -> &'static str {
186 match self {
187 Wire::Ssh => "ssh",
188 Wire::Http => "smart http",
189 }
190 }
191}
192
193/// **What transport `url` names, or a refusal that says what gunnar can dial.**
194///
195/// The scp-like form is recognised the way git recognises it: a `:` before any
196/// `/`, with no scheme. A URL that is neither is refused rather than guessed
197/// at — a mirror registered against a path on the local disk would otherwise
198/// look like it was configured and then fail on the first sync with a
199/// connection error.
200pub fn classify(url: &str) -> Result<Wire, String> {
201 let url = url.trim();
202 if url.starts_with("ssh://") || url.starts_with("git+ssh://") {
203 return Ok(Wire::Ssh);
204 }
205 if url.starts_with("https://") || url.starts_with("http://") {
206 return Ok(Wire::Http);
207 }
208 if url.contains("://") {
209 return Err(format!(
210 "{} names a scheme gunnar cannot mirror from. The two transports are `ssh://` \
211 and `https://`",
212 scrub(url)
213 ));
214 }
215 // `git@host:ns/repo.git` — a colon before any slash, and a host in front
216 // of it.
217 match (url.find(':'), url.find('/')) {
218 (Some(colon), slash) if colon > 0 && slash.is_none_or(|slash| colon < slash) => {
219 Ok(Wire::Ssh)
220 }
221 _ => Err(format!(
222 "{url:?} is not a remote gunnar can mirror from. Write it as \
223 `ssh://git@host/ns/repo.git`, `git@host:ns/repo.git` or `https://host/ns/repo.git`"
224 )),
225 }
226}
227
228// ── where a mirror may dial ──────────────────────────────────────────────────
229
230/// **The host and port a mirror URL names**, in the three shapes
231/// [`classify`] admits.
232///
233/// Deliberately not a URL parser: the only question asked of this string is
234/// *which machine would be dialled*, and the answer is the authority. A
235/// userinfo half is dropped — `git@host` names `host` — and an IPv6 literal
236/// keeps its brackets off so it can be parsed as an address.
237pub(crate) fn authority(url: &str) -> Result<(String, u16), String> {
238 let url = url.trim();
239 let (default_port, rest) = if let Some(rest) = url.strip_prefix("ssh://") {
240 (22, rest)
241 } else if let Some(rest) = url.strip_prefix("git+ssh://") {
242 (22, rest)
243 } else if let Some(rest) = url.strip_prefix("https://") {
244 (443, rest)
245 } else if let Some(rest) = url.strip_prefix("http://") {
246 (80, rest)
247 } else {
248 // The scp-like `git@host:ns/repo.git`. `classify` has already decided
249 // this shape is ssh; the colon here separates a PATH, never a port.
250 // The userinfo goes first so a `@` inside it cannot be read as a host,
251 // and a bracketed literal is taken whole — `[::1]:path` is all colons
252 // and splitting on the first one would yield `[`.
253 let rest = url.rsplit('@').next().unwrap_or_default();
254 let host = match rest.strip_prefix('[') {
255 Some(inside) => inside.split(']').next().unwrap_or_default(),
256 None => rest.split(':').next().unwrap_or_default(),
257 };
258 return non_empty(host, url).map(|host| (host, 22));
259 };
260 let authority = rest.split(['/', '?', '#']).next().unwrap_or_default();
261 let authority = authority.rsplit('@').next().unwrap_or_default();
262 // `[::1]:9418` and `[::1]`, before the bare-host split, because an IPv6
263 // literal is all colons.
264 if let Some(rest) = authority.strip_prefix('[') {
265 let Some((host, tail)) = rest.split_once(']') else {
266 return Err(format!(
267 "{} names an unterminated IPv6 literal",
268 scrub(url)
269 ));
270 };
271 let port = match tail.strip_prefix(':') {
272 None => default_port,
273 Some(digits) => digits.parse().map_err(|_| {
274 format!(
275 "{} names a port that is not a number",
276 scrub(url)
277 )
278 })?,
279 };
280 return non_empty(host, url).map(|host| (host, port));
281 }
282 match authority.split_once(':') {
283 None => non_empty(authority, url).map(|host| (host, default_port)),
284 Some((host, digits)) => {
285 let port = digits.parse().map_err(|_| {
286 format!(
287 "{} names a port that is not a number",
288 scrub(url)
289 )
290 })?;
291 non_empty(host, url).map(|host| (host, port))
292 }
293 }
294}
295
296fn non_empty(host: &str, url: &str) -> Result<String, String> {
297 if host.is_empty() {
298 return Err(format!(
299 "{} names no host to dial",
300 scrub(url)
301 ));
302 }
303 Ok(host.to_owned())
304}
305
306/// **Refuse a destination this server can reach and its caller cannot** — the
307/// pure half, with the addresses handed in.
308///
309/// `Mirror.Add` needs Admin and `Mirror.Sync` needs Write, but *every enrolled
310/// principal is intrinsically Admin in its own namespace*
311/// (`AuthTable::owns_account`), so `Mirror.Add{create_repo}` against
312/// `alice/anything` is available to anybody with a key. Before this check the
313/// only thing standing between that caller and an arbitrary request from the
314/// server's own network position was `remote_url.trim().is_empty()`: the cloud
315/// metadata endpoint at `169.254.169.254`, a control plane on `127.0.0.1`, a
316/// printer on `10.0.0.0/8` — all dialled by the appliance, with the appliance's
317/// routes, and the outcome partly readable back through `Mirror.Status`.
318///
319/// **Every** resolved address has to be routable, not merely the first: a name
320/// that resolves to one public address and one loopback address is the
321/// rebinding shape, and taking the best of the two is how a check like this
322/// gets to be decorative.
323///
324/// The ranges are written out rather than taken from `IpAddr::is_global`, which
325/// is unstable, and each line is the reason it is there.
326pub(crate) fn address_verdict(host: &str, addrs: &[std::net::IpAddr]) -> Result<(), String> {
327 if addrs.is_empty() {
328 return Err(format!("{host} resolves to no address at all"));
329 }
330 for addr in addrs {
331 if let Some(why) = not_routable(*addr) {
332 return Err(format!(
333 "{host} resolves to {addr}, which is {why}. A mirror dials from the server, so a \
334 destination the caller could not reach on their own is the server's network \
335 position being borrowed. Create `{MIRROR_ALLOW_PRIVATE}` under this box's data \
336 root to allow it deliberately."
337 ));
338 }
339 }
340 Ok(())
341}
342
343/// Why `addr` is not somewhere an arbitrary caller should be able to point this
344/// server, or `None` when it is an ordinary public address.
345fn not_routable(addr: std::net::IpAddr) -> Option<&'static str> {
346 use std::net::IpAddr;
347 match addr {
348 IpAddr::V4(v4) => {
349 let [a, b, c, _] = v4.octets();
350 match () {
351 _ if v4.is_unspecified() => Some("the unspecified address"),
352 _ if v4.is_loopback() => Some("loopback — this server itself"),
353 _ if v4.is_private() => Some("a private network (RFC 1918)"),
354 _ if v4.is_link_local() => Some(
355 "link-local (RFC 3927), the range cloud metadata services answer on",
356 ),
357 _ if v4.is_broadcast() => Some("the broadcast address"),
358 _ if v4.is_multicast() => Some("multicast"),
359 _ if v4.is_documentation() => Some("a documentation range (RFC 5737)"),
360 // 100.64.0.0/10, carrier-grade NAT: the provider's own network,
361 // not the caller's.
362 _ if a == 100 && (64..128).contains(&b) => Some("carrier-grade NAT (RFC 6598)"),
363 // 192.0.0.0/24, IETF protocol assignments.
364 _ if a == 192 && b == 0 && c == 0 => Some("an IETF protocol assignment"),
365 // 198.18.0.0/15, benchmarking.
366 _ if a == 198 && (b == 18 || b == 19) => Some("a benchmarking range (RFC 2544)"),
367 // 240.0.0.0/4, reserved.
368 _ if a >= 240 => Some("a reserved range"),
369 _ => None,
370 }
371 }
372 // A v4 address written the long way is the same address. Checked
373 // first, or `::ffff:127.0.0.1` walks straight past every arm below it.
374 IpAddr::V6(v6) if v6.to_ipv4_mapped().is_some() => {
375 not_routable(IpAddr::V4(v6.to_ipv4_mapped().expect("just matched")))
376 }
377 IpAddr::V6(v6) => {
378 let segments = v6.segments();
379 match () {
380 _ if v6.is_unspecified() => Some("the unspecified address"),
381 _ if v6.is_loopback() => Some("loopback — this server itself"),
382 _ if v6.is_multicast() => Some("multicast"),
383 // fc00::/7, unique local.
384 _ if segments[0] & 0xfe00 == 0xfc00 => Some("a unique-local address (RFC 4193)"),
385 // fe80::/10, link-local.
386 _ if segments[0] & 0xffc0 == 0xfe80 => Some("link-local (RFC 4291)"),
387 // 2001:db8::/32, documentation.
388 _ if segments[0] == 0x2001 && segments[1] == 0x0db8 => {
389 Some("a documentation range (RFC 3849)")
390 }
391 _ => None,
392 }
393 }
394 }
395}
396
397/// [`address_verdict`] with the resolution and the opt-out file read — the
398/// edge, and the only part of this that touches the world.
399///
400/// # The window this does not close, said rather than glossed
401///
402/// The name is resolved here and again by `ureq` or by the ssh dialler a moment
403/// later, so a resolver that answers differently the second time (DNS
404/// rebinding) is not stopped by this. Closing that needs the dial pinned to the
405/// address that was checked, which is a change inside `ureq`'s and
406/// `nornir-git`'s connect paths rather than in this file. What this does close
407/// is the whole of the *static* case — a URL that simply names an internal
408/// address — which is what `Mirror.Add` accepted unchecked.
409pub fn check_destination(url: &str, data_root: &std::path::Path) -> Result<(), String> {
410 use std::net::ToSocketAddrs as _;
411 if data_root.join(MIRROR_ALLOW_PRIVATE).exists() {
412 return Ok(());
413 }
414 let (host, port) = authority(url)?;
415 // A literal needs no resolver, and asking one for it would let a DNS server
416 // answer a question that has no question in it.
417 if let Ok(addr) = host.parse::<std::net::IpAddr>() {
418 return address_verdict(&host, &[addr]);
419 }
420 let addrs: Vec<std::net::IpAddr> = (host.as_str(), port)
421 .to_socket_addrs()
422 .map_err(|err| format!("{host} could not be resolved: {err}"))?
423 .map(|socket| socket.ip())
424 .collect();
425 address_verdict(&host, &addrs)
426}
427
428/// **Refuse a credential that cannot be offered over the transport the URL
429/// names — by name, before anything is dialled.**
430///
431/// The failure this prevents is the quiet one. An SSH key against an `https://`
432/// remote is not "authentication failed": there is nowhere in an HTTP request
433/// to put a private key, so the request goes out anonymous and the upstream
434/// answers 401 — and the operator, who can see that a credential IS stored,
435/// concludes the key is wrong and rotates it. Forever. The same in reverse: a
436/// token has no place in an SSH publickey exchange, so the dial falls through
437/// to whatever else is offerable and fails on authentication with no mention of
438/// the token at all.
439///
440/// A stored credential of no kind is not a mismatch — a public upstream mirrors
441/// fine with nothing, and that path must keep working.
442pub fn check_credential(wire: Wire, credential: &Credential) -> Result<(), String> {
443 match (wire, credential) {
444 (_, Credential::None) => Ok(()),
445 (Wire::Ssh, Credential::SshKey(_)) => Ok(()),
446 (Wire::Http, Credential::Token { .. }) => Ok(()),
447 (wire, held) => Err(format!(
448 "this mirror's remote is dialled over {}, and the credential stored for it is {}. \
449 {} cannot carry it, so the fetch would go out unauthenticated and be refused as \
450 though no credential were set. Store {} instead, with `Mirror.SetCredential`.",
451 wire.name(),
452 held.kind().name(),
453 wire.name(),
454 match wire {
455 Wire::Ssh => "an ssh private key",
456 Wire::Http => "a token",
457 }
458 )),
459 }
460}
461
462// ── the gate ─────────────────────────────────────────────────────────────────
463
464/// **The repository's branch protection, over the batch a mirror is about to
465/// apply.**
466///
467/// One implementation: this is `gunnar_policy::BranchProtectionGate`, the same
468/// object `receive_pack`'s [`crate::transport::PolicyGate`] evaluates, reading
469/// the same document through the same [`crate::transport::load_policy`]. A
470/// second copy of "what does `allow_force_push` mean" is precisely the pair
471/// that drifts, and the three independent ways past branch protection this
472/// repository has had are all the same shape: a second writer that does what
473/// the gated writer does, without the gate.
474///
475/// # What a mirror can decide, and the one thing it cannot
476///
477/// `allow_create`, `allow_delete` and `allow_force_push` are decidable from the
478/// batch and the objects, which are already in the store when [`apply`] runs.
479/// `require_signature` is decidable too — but only against the set of objects
480/// the fetch introduced, so a repository carrying any protection rule makes the
481/// ingest ask for that set ([`crate::storage::ObjectSet::Required`]) instead of
482/// paying nothing for it, exactly as a push does when a gate is live.
483///
484/// `allowed_principals` is the one that cannot be decided here, and it is
485/// refused rather than skipped. The caller's identity does not cross the
486/// control-plane seam on this verb — `mirror_sync(&self, path)` carries a
487/// repository and nothing about who asked — so there is no principal to compare
488/// against the list. "I could not tell" is not "it is fine", so a rule that
489/// names its principals refuses the whole sync by name, and the operator either
490/// drops the rule or stops mirroring onto the refs it covers.
491struct Protection {
492 gate: gunnar_policy::BranchProtectionGate,
493 keys: gunnar_policy::StaticKeyDirectory,
494 repo: gunnar_policy::RepoId,
495}
496
497impl Protection {
498 /// The rules covering `repo`, or `None` when it has none configured.
499 ///
500 /// Read **before** the fetch, because whether any rule exists is what
501 /// decides if the ingest pays for an object-set enumeration — the same
502 /// ordering, for the same reason, as `receive_pack`'s.
503 ///
504 /// `Err` is a policy document that could not be read, and it REFUSES the
505 /// sync — the same fail-closed rule the push path has (GUN-007). A mirror
506 /// that synced with its gates disabled because the policy did not parse
507 /// would be the bypass GUN-032 closed, back through the side door.
508 fn open(
509 stores: &ServerStores,
510 row: &gunnar_core::Repo,
511 git_dir: &std::path::Path,
512 ) -> std::result::Result<Option<Self>, String> {
513 let config = crate::transport::load_policy(
514 stores,
515 row,
516 &git_dir.join(crate::transport::POLICY_FILE),
517 )
518 .map_err(|why| format!("refusing to mirror into {}: {why}", row.key()))?;
519 if config.protection.is_empty() {
520 return Ok(None);
521 }
522 Ok(Some(Protection {
523 gate: gunnar_policy::BranchProtectionGate::new(config.protection.clone()),
524 keys: crate::transport::key_directory(stores),
525 repo: gunnar_policy::RepoId::new(&row.namespace, &row.name),
526 }))
527 }
528
529 /// **Refuse the batch, or let it through.** `Err` is the refusal a FAILED
530 /// job carries, and nothing has been written when it is returned.
531 ///
532 /// `new_objects` is the fetch's own receipt. An ingest that did not
533 /// enumerate hands back the reason rather than an empty set, and an empty
534 /// set here would be a silent always-allow for `require_signature` — so the
535 /// `Err` arm refuses, quoting why.
536 fn check(
537 &self,
538 git: &dyn gunnar_store::git_storage_trait::GitServe,
539 updates: &[gunnar_policy::RefUpdate],
540 new_objects: std::result::Result<&[ObjectId], &str>,
541 ) -> Result<(), String> {
542 use gunnar_policy::Gate as _;
543
544 // The field with no answer on this path, refused first and by name.
545 for update in updates {
546 for rule in self.gate.rules() {
547 if !rule.pattern.matches_ref(&update.name) {
548 continue;
549 }
550 if rule.allowed_principals.is_some() {
551 return Err(format!(
552 "{} is protected by {}, which names the principals that may move it. A \
553 mirror sync carries no principal — the control-plane verb takes a \
554 repository and not a caller — so that rule cannot be satisfied and this \
555 sync is refused rather than applied unchecked. Nothing was moved.",
556 update.name, rule.pattern
557 ));
558 }
559 break;
560 }
561 }
562
563 let new_objects = new_objects.map_err(|reason| {
564 format!(
565 "this repository has branch protection and the fetched pack's object set could \
566 not be enumerated ({reason}), so the gate could not decide. Nothing was moved."
567 )
568 })?;
569 // Never consulted: every rule that could read it has already refused
570 // above. Named rather than left as a plausible-looking value.
571 let principal = gunnar_policy::Principal::machine("mirror");
572 let ctx = gunnar_policy::PushContext::new(
573 &principal,
574 &self.repo,
575 updates,
576 new_objects,
577 git,
578 &self.keys,
579 );
580 match self.gate.check(&ctx) {
581 Ok(gunnar_policy::Verdict::Allow) => Ok(()),
582 Ok(gunnar_policy::Verdict::Reject(rejection)) => Err(format!(
583 "branch protection refused this mirror's reference batch: {}. Nothing was moved.",
584 rejection.reason
585 )),
586 // A gate that could not decide refuses; it does not allow.
587 Err(err) => Err(format!(
588 "branch protection could not decide this mirror's reference batch: {err}. \
589 Nothing was moved."
590 )),
591 }
592 }
593}
594
595// ── the local half ───────────────────────────────────────────────────────────
596
597/// **What this repository already holds**, as `fetch_pack`'s have-source.
598///
599/// One `GitOps::refs()` call, one snapshot, taken inside `FetchPack::fetch` so
600/// the `have` list and every `old` in the plan come from the same read — a
601/// reference that moves under the fetch cannot make the plan promise an old
602/// value the store never held.
603///
604/// `HEAD` and the pseudo-refs are not rows in `refs()` at all, and a purely
605/// symbolic row under `refs/` carries no oid and is skipped: it is not a thing
606/// to offer as a `have`, and `fetch_pack`'s own doc says an unborn ref should
607/// simply be absent.
608struct StoreRefs<'a> {
609 git: &'a dyn gunnar_store::git_storage_trait::GitServe,
610 /// Objects this store holds that no reference names yet — an interrupted
611 /// import's ([`already_here`]). Offered as `have`s under names the plan
612 /// never matches, so they change what is sent and nothing that is written.
613 extra_haves: Vec<ObjectId>,
614}
615
616impl LocalRefs for StoreRefs<'_> {
617 fn tips(&self) -> gunnar_wire::Result<Vec<(String, ObjectId)>> {
618 let rows = GitOps::refs(self.git).map_err(|err| {
619 gunnar_wire::Error::Store(gunnar_store::Error::backend(format!(
620 "reading this repository's refs to offer them as haves: {err:#}"
621 )))
622 })?;
623 let mut out = Vec::with_capacity(rows.len());
624 for row in rows {
625 if row.name == "HEAD" {
626 continue;
627 }
628 let Some(raw) = row.oid.as_deref() else {
629 continue;
630 };
631 let Ok(oid) = gix_hash::oid::try_from_bytes(raw) else {
632 return Err(gunnar_wire::Error::Store(gunnar_store::Error::backend(
633 format!("{} holds an oid this store cannot decode", row.name),
634 )));
635 };
636 out.push((row.name, oid.to_owned()));
637 }
638 for (n, oid) in self.extra_haves.iter().enumerate() {
639 out.push((format!("refs/gunnar/resume/{n}"), *oid));
640 }
641 Ok(out)
642 }
643}
644
645/// **The queue's progress field, written from the wire and from the walk.**
646///
647/// Three publishers, one job record, and each of the three is the only place on
648/// the system that knows its own number:
649///
650/// | published by | what | when |
651/// |---|---|---|
652/// | [`FetchProgress::objects_total`] | the denominator | once, the instant the packfile header is read |
653/// | [`FetchProgress::pack_bytes`] | the byte count | as the stream is consumed, while the network is the bottleneck |
654/// | [`PackProgress::indexed`] | the numerator | once per object, as the engine's pack walk splits them |
655///
656/// # Why the numerator arrives from the other side of the seam
657///
658/// `gunnar-wire` reads bytes 8..12 of the header and can publish `(0, N)`
659/// before an object is stored — twelve bytes, and it is the reason the
660/// denominator is live at all. It cannot publish the numerator: counting
661/// objects as they pass means inflating every one of them, which is a second
662/// pack decoder, and gunnar has exactly one (one implementation, and `fetch_pack`'s own
663/// module doc refuses it in those words). The one decoder is the storage
664/// engine's walk, which has to split every entry anyway, so the count is free
665/// exactly there and nowhere else. This type is where the two halves meet.
666///
667/// # `total` is remembered here, and that is not a second counter
668///
669/// [`PackProgress::indexed`] carries its own `total` — the count the engine
670/// read from the same header — and it is *checked* against the one the wire
671/// announced rather than trusted, because two readings of one field that
672/// disagree mean the ingest is being handed a different pack from the one that
673/// was negotiated. `fetch_pack` makes the same comparison at the end, on the
674/// receipt; this is the same fact asserted while there is still a transfer to
675/// abandon.
676struct QueueProgress<'a> {
677 report: &'a Reporter<Mirror>,
678 total: AtomicU64,
679 /// A second sink, for a caller that has to see the *series* rather than the
680 /// job's latest reading. See [`sync_once_watching`].
681 watching: Option<&'a dyn git_storage_trait::PackProgress>,
682}
683
684impl QueueProgress<'_> {
685 /// **The one place a sync's progress pair is written.**
686 ///
687 /// All three publishers go through here — the header's `(0, N)`, the
688 /// engine's per-object readings, and [`finish`]'s `(N, N)` — so the job and
689 /// any watcher see the same values in the same order. Two routes to the
690 /// same field would be two places the series could differ, and the series
691 /// is the whole subject.
692 fn publish(&self, done: u64, total: u64) {
693 self.report.progress(done, total);
694 if let Some(watcher) = self.watching {
695 watcher.indexed(done, total);
696 }
697 }
698}
699
700impl FetchProgress for QueueProgress<'_> {
701 fn objects_total(&self, objects: u32) {
702 self.total.store(u64::from(objects), Ordering::SeqCst);
703 self.publish(0, u64::from(objects));
704 }
705
706 fn pack_bytes(&self, streamed: u64) {
707 self.report.report(|mirror| mirror.bytes = streamed);
708 }
709}
710
711impl git_storage_trait::PackProgress for QueueProgress<'_> {
712 fn indexed(&self, done: u64, total: u64) {
713 // The wire's denominator wins. An engine that read a different total
714 // from the same twelve bytes is a bug worth a log line, not a reason to
715 // renumber a progress bar mid-flight and leave a console watching a
716 // percentage that jumped sideways.
717 let announced = self.total.load(Ordering::SeqCst);
718 let total = if announced == 0 || announced == total {
719 total
720 } else {
721 tracing::warn!(
722 target: TARGET_GIT,
723 announced,
724 indexed_total = total,
725 "the packfile header the wire read and the one the store read declare \
726 different object counts; the wire's is the one being rendered"
727 );
728 announced
729 };
730 self.publish(done, total);
731 }
732}
733
734// ── the fetch ────────────────────────────────────────────────────────────────
735
736/// **One sync, start to finish.** The [`crate::mirror_queue::MirrorRunner`]
737/// body.
738///
739/// Errors are `String` because they are going onto a job a console will read,
740/// not up a call stack — the same reason `gc_queue`'s runner returns one.
741pub fn sync_once(
742 stores: &Arc<ServerStores>,
743 order: &MirrorOrder,
744 credential: &Credential,
745 report: &Reporter<Mirror>,
746) -> Result<(), String> {
747 sync_once_watching(stores, order, credential, report, None)
748}
749
750/// [`sync_once`], with a **second** place the object count is published to.
751///
752/// # Why this door exists, and why it is not a test hook
753///
754/// [`Reporter::progress`] *overwrites*, by design — its own doc says a progress
755/// that accumulated would be a second counter to keep right — so the job record
756/// holds the latest reading and never the series. That is correct for a console,
757/// which only ever wants the latest reading, and it means the one thing that
758/// cannot be established from the job record afterwards is **whether the count
759/// moved at all**. A publisher that fires `(0, N)` and then `(N, N)` and a
760/// publisher that fires five hundred times in between leave the job in exactly
761/// the same state, so from the outside they are the same publisher.
762///
763/// That is measure-never-assert in its plainest form: a stub is
764/// indistinguishable from an
765/// implementation unless something can see it work — and it is precisely the
766/// defect this seam was opened to close: `(0, N) → (N, N)` was the shape for as
767/// long as no engine published from inside its pack walk. So the series gets a
768/// door, and the acceptance test stands at it. There is one other caller,
769/// [`sync_once`], which passes `None`.
770///
771/// The sink is *additional*: the job is written exactly as before, in the same
772/// order, whether or not anybody is watching. A test that changed what the
773/// production path publishes would be testing itself.
774pub fn sync_once_watching(
775 stores: &Arc<ServerStores>,
776 order: &MirrorOrder,
777 credential: &Credential,
778 report: &Reporter<Mirror>,
779 watching: Option<&dyn git_storage_trait::PackProgress>,
780) -> Result<(), String> {
781 let wire = classify(&order.upstream)?;
782 check_credential(wire, credential)?;
783 // Checked again on the worker, and not only at the door where
784 // `Mirror.Add` checked it: a registration made before this rule existed,
785 // or a name whose A record has moved since, reaches the dial through here
786 // and nowhere else.
787 check_destination(&order.upstream, stores.root())?;
788
789 // Resolved here, on the worker, and not at enqueue time: a repository can
790 // be deleted while its sync waits, and discovering that when the work
791 // starts is a FAILED job saying so, which is the truth.
792 let repo_ref = gunnar_core::resolve_repo_path(&order.path)
793 .map_err(|err| format!("`{}` is not a valid <namespace>/<name>: {err:#}", order.path))?;
794 let existing = stores
795 .catalog()
796 .get(&repo_ref)
797 .map_err(|err| format!("reading the catalog row for {}: {err:#}", order.path))?;
798 // **An import's repository is made HERE, by the first sync that reaches
799 // the upstream** (owner order 2026-10-18; `MirrorRecord::creates_repo`).
800 let to_create = existing.is_none()
801 && stores
802 .mirrors()
803 .get(&repo_ref)
804 .map_err(|err| format!("reading the registration for {}: {err:#}", order.path))?
805 .is_some_and(|record| record.creates_repo);
806 if existing.is_none() && !to_create {
807 return Err(format!("{} is no longer a live repository", order.path));
808 }
809 // **An encrypted repository is never fetched into** (wiring tracer B
810 // 2026-10-08): the ingest below runs no encrypted gate (`PushIngest::new`),
811 // so an upstream's plaintext would be stored and framed for the twin.
812 // `Mirror.Add` / `Repos.SetEncrypted` refuse making such a mirror; this is
813 // a registration older than that rule, or a flag set below the door.
814 if existing.as_ref().is_some_and(|row| row.encrypted) {
815 return Err(format!(
816 "{} is encrypted, so it is not fetched into: a mirror copies another server's \
817 plaintext, and an encrypted repository holds only what its members sealed \
818 (remove the mirror)",
819 order.path
820 ));
821 }
822 // **The namespace's room, asked of the same gates a push is held to**
823 // (2026-10-02). A suspended namespace or one already at its cap fetches
824 // nothing — refused here, before the dial, in the gate's own words — and
825 // the fetch is held to the tightest ceiling in the chain as it streams.
826 let ceiling = room_for(&stores.plan_gates(), repo_ref.namespace()).map_err(|(why, line)| {
827 report.report(|r| r.refused = why);
828 line
829 })?;
830
831 let auth = match credential {
832 Credential::Token { username, secret } if username.is_empty() => {
833 Some(format!("Bearer {secret}"))
834 }
835 Credential::Token { username, secret } => {
836 Some(format!("Basic {}", base64_basic(username, secret)))
837 }
838 _ => None,
839 };
840
841 let row = match existing {
842 Some(row) => row,
843 None => create_from_upstream(stores, &repo_ref, order, wire, credential, auth.clone())?,
844 };
845 let fetched = fetch_into(stores, &row, order, wire, credential, auth, ceiling, report, watching)
846 .and_then(|advertised| landed(stores, &row, order, &advertised).map(|()| advertised));
847 match &fetched {
848 Ok(advertised) => after_landing(stores, &repo_ref, &row, order, advertised),
849 Err(_) if to_create => discard_created(stores, &repo_ref, order),
850 Err(_) => {}
851 }
852 fetched.map(|_| ())
853}
854
855/// **A sync is DONE only when what the upstream advertised is here**
856/// (import-big, 2026-10-26).
857///
858/// Every branch and tag the upstream advertised must be a reference in this
859/// repository when the sync says it is done, and an upstream that advertised
860/// none is not an import that worked: it is an empty import, said as a
861/// failure. Read back from the store, after the apply, rather than inferred
862/// from the plan — the plan is what was meant, this is what landed. Before it,
863/// a fetch whose plan came out empty applied nothing and settled DONE over a
864/// repository with no references at all.
865fn landed(
866 stores: &Arc<ServerStores>,
867 row: &gunnar_core::model::Repo,
868 order: &MirrorOrder,
869 advertised: &[gunnar_wire::upload_pack::AdvertisedRef],
870) -> Result<(), String> {
871 let wanted: Vec<&str> = advertised
872 .iter()
873 .filter(|r| fetch_pack::mirror(&r.name) && r.target.is_some())
874 .map(|r| r.name.as_str())
875 .collect();
876 let upstream = scrub(&order.upstream);
877 if wanted.is_empty() {
878 return Err(format!(
879 "{upstream} has no branches or tags, so there was nothing to import into {}. \
880 Check the address, or push a first commit there and sync again.",
881 order.path
882 ));
883 }
884 let store = stores
885 .repo_store(row)
886 .map_err(|err| format!("opening {} to check what landed: {err:#}", order.path))?;
887 let held = GitOps::refs(store.serve())
888 .map_err(|err| format!("reading {}'s references to check what landed: {err:#}", order.path))?;
889 let missing: Vec<&str> = wanted
890 .iter()
891 .copied()
892 .filter(|name| !held.iter().any(|r| r.name == *name && r.oid.is_some()))
893 .collect();
894 if let Some(first) = missing.first() {
895 return Err(format!(
896 "the fetch from {upstream} ended without its references landing: {} of {} \
897 (first {first}) are not in {}. Nothing is reported as imported that is not \
898 here; sync again.",
899 missing.len(),
900 wanted.len(),
901 order.path
902 ));
903 }
904 Ok(())
905}
906
907/// **Ask the upstream what it is, then make the repository to match.**
908///
909/// One `ls-refs` at the upstream's own object format (`ls_remote_any`): the
910/// format becomes the repository's, and `HEAD`'s `symref-target` its default
911/// branch. Nothing is created when the upstream cannot be reached or refuses
912/// (a private repository without its token), so a failed first import leaves
913/// no repository behind.
914fn create_from_upstream(
915 stores: &Arc<ServerStores>,
916 repo_ref: &gunnar_core::RepoRef,
917 order: &MirrorOrder,
918 wire: Wire,
919 credential: &Credential,
920 auth: Option<String>,
921) -> Result<gunnar_core::model::Repo, String> {
922 let advertisement = ls_remote(stores, order, wire, credential, auth)?;
923 let hash = match advertisement.hash_kind {
924 gix_hash::Kind::Sha1 => gunnar_core::HashKind::Sha1,
925 _ => gunnar_core::HashKind::Sha256,
926 };
927 let mut new = gunnar_core::NewRepo::new(repo_ref.clone())
928 .hash_kind(hash)
929 .visibility(gunnar_core::Visibility::Private);
930 new.description = format!("mirror of {}", scrub(&order.upstream));
931 if let Some(branch) = advertised_head(&advertisement.refs) {
932 new = new.default_branch(branch);
933 }
934 tracing::info!(
935 target: TARGET_GIT,
936 event = "mirror.create_from_upstream",
937 repo = %order.path,
938 hash = ?hash,
939 "an import's repository is made at the upstream's object format and default branch"
940 );
941 let row = stores
942 .create_repo(new, false)
943 .map_err(|err| format!("creating {} for the import: {err:#}", order.path))?;
944 stores
945 .repo_store(&row)
946 .map_err(|err| format!("opening {} to mirror into it: {err:#}", order.path))?;
947 Ok(row)
948}
949
950/// **One `ls-refs` against the upstream**, at its own object format.
951fn ls_remote(
952 stores: &Arc<ServerStores>,
953 order: &MirrorOrder,
954 wire: Wire,
955 credential: &Credential,
956 auth: Option<String>,
957) -> Result<gunnar_wire::fetch_pack::Advertisement, String> {
958 let listed = |err: gunnar_wire::Error| {
959 format!(
960 "asking {} what it holds: {err}",
961 scrub(&order.upstream)
962 )
963 };
964 match wire {
965 Wire::Http => {
966 let (mut reader, mut writer) = open_http(&order.upstream, auth)?;
967 gunnar_wire::ls_remote_any(&mut reader, &mut writer).map_err(listed)
968 }
969 Wire::Ssh => dial_ssh(
970 &order.upstream,
971 credential,
972 &stores.root().join(MIRROR_KNOWN_HOSTS),
973 strict_host_keys(stores.root()),
974 |reader, writer| gunnar_wire::ls_remote_any(reader, writer),
975 )?
976 .map_err(listed),
977 }
978}
979
980/// **What an interrupted import already brought in**, offered back to the
981/// upstream as `have`s (import-big, 2026-10-26).
982///
983/// A repository that holds no branch and no tag is either new or an import
984/// that stopped between its pack landing and its references being written —
985/// a restart in the index, the longest step of a big import. The pack's bytes
986/// were durable before that step, and the indexer resumes them on its own after
987/// the restart; what was missing is only the reference write. So the resumed
988/// sync asks the upstream what it advertises, lets the index catch up, and
989/// offers every advertised tip this store already has as a `have`: the
990/// upstream then sends nothing it sent before, and a 6.6 GB kernel pack is not
991/// fetched (or stored) a second time. A new repository has nothing, and pays
992/// one `ls-refs` for finding out.
993#[allow(clippy::too_many_arguments)]
994fn already_here(
995 stores: &Arc<ServerStores>,
996 git: &dyn gunnar_store::git_storage_trait::GitServe,
997 git_dir: &std::path::Path,
998 order: &MirrorOrder,
999 wire: Wire,
1000 credential: &Credential,
1001 auth: Option<String>,
1002 report: &Reporter<Mirror>,
1003) -> Result<Vec<ObjectId>, String> {
1004 let held = GitOps::refs(git).map_err(|err| format!("reading {}'s references: {err:#}", order.path))?;
1005 if held.iter().any(|r| fetch_pack::mirror(&r.name) && r.oid.is_some()) {
1006 return Ok(Vec::new());
1007 }
1008 let advertisement = ls_remote(stores, order, wire, credential, auth)?;
1009 index_landed(git, git_dir, stores.engine(), report).map_err(|err| {
1010 format!("{} holds objects this server could not index: {err}", order.path)
1011 })?;
1012 let mut out = Vec::new();
1013 for advertised in &advertisement.refs {
1014 let Some(target) = advertised.target else { continue };
1015 if !fetch_pack::mirror(&advertised.name) || out.contains(&target) {
1016 continue;
1017 }
1018 if GitOps::has(git, target.as_bytes()).unwrap_or(false) {
1019 out.push(target);
1020 }
1021 }
1022 if !out.is_empty() {
1023 tracing::info!(
1024 target: TARGET_GIT,
1025 event = "mirror.sync.resumed",
1026 repo = %order.path,
1027 haves = out.len(),
1028 "an import that stopped after its pack landed resumes without fetching it again"
1029 );
1030 }
1031 Ok(out)
1032}
1033
1034/// `HEAD`'s branch, as the upstream advertised it, when it is a branch.
1035fn advertised_head(refs: &[gunnar_wire::upload_pack::AdvertisedRef]) -> Option<String> {
1036 refs.iter()
1037 .find(|r| r.name == "HEAD")
1038 .and_then(|r| r.symref_target.clone())
1039 .filter(|target| target.starts_with("refs/heads/") && gunnar_store::refname::validate(target).is_ok())
1040}
1041
1042/// A first import that failed after its repository was made: the repository
1043/// goes again, so nothing empty is left behind. Best effort — a removal that
1044/// fails is logged, and the registration still says the repository is to come.
1045fn discard_created(stores: &Arc<ServerStores>, repo_ref: &gunnar_core::RepoRef, order: &MirrorOrder) {
1046 let removed = stores
1047 .catalog()
1048 .delete(repo_ref)
1049 .map_err(|err| format!("{err:#}"))
1050 .and_then(|_| stores.purge_repo(repo_ref).map_err(|err| format!("{err:#}")));
1051 if let Err(err) = removed {
1052 tracing::warn!(
1053 target: TARGET_GIT,
1054 repo = %order.path,
1055 %err,
1056 "a failed first import could not remove the repository it had made"
1057 );
1058 }
1059}
1060
1061/// **What a landed sync owes the mirror besides its references**: the
1062/// upstream's default branch (when it holds that branch now) and whether the
1063/// upstream uses Git LFS. Neither fails the sync — the references landed.
1064fn after_landing(
1065 stores: &Arc<ServerStores>,
1066 repo_ref: &gunnar_core::RepoRef,
1067 row: &gunnar_core::model::Repo,
1068 order: &MirrorOrder,
1069 advertised: &[gunnar_wire::upload_pack::AdvertisedRef],
1070) {
1071 let mut row = row.clone();
1072 if let Some(branch) = advertised_head(advertised) {
1073 let held = stores
1074 .repo_store(&row)
1075 .ok()
1076 .and_then(|store| GitOps::refs(store.serve()).ok())
1077 .is_some_and(|refs| refs.iter().any(|r| r.name == branch && r.oid.is_some()));
1078 if held && branch != row.default_branch {
1079 match stores.catalog().set_default_branch(repo_ref, &branch) {
1080 Ok(moved) => {
1081 row = moved;
1082 if let Err(err) = stores.repoint_head(&row) {
1083 tracing::warn!(target: TARGET_GIT, repo = %order.path, %err, "pointing HEAD at the upstream's default branch");
1084 }
1085 }
1086 Err(err) => {
1087 tracing::warn!(target: TARGET_GIT, repo = %order.path, %err, "taking the upstream's default branch");
1088 }
1089 }
1090 }
1091 }
1092 let lfs = uses_lfs(stores, &row);
1093 if let Err(err) = stores.mirrors().set_lfs_detected(repo_ref, lfs) {
1094 tracing::warn!(target: TARGET_GIT, repo = %order.path, %err, "recording whether the upstream uses LFS");
1095 }
1096}
1097
1098/// **Does the default branch route files through Git LFS?** A `filter=lfs`
1099/// in its root `.gitattributes` is the sign every LFS repository carries; the
1100/// files it names arrive as pointer files, and their content is not mirrored.
1101fn uses_lfs(stores: &Arc<ServerStores>, row: &gunnar_core::model::Repo) -> bool {
1102 use gunnar_store::git_storage_trait::ObjType;
1103 use gunnar_store::object;
1104 let Ok(store) = stores.repo_store(row) else {
1105 return false;
1106 };
1107 let git = store.serve();
1108 let hash = store.hash_kind();
1109 let Some(tip) = GitOps::refs(git)
1110 .ok()
1111 .and_then(|refs| refs.into_iter().find(|r| r.name == row.default_branch))
1112 .and_then(|r| r.oid)
1113 .and_then(|raw| ObjectId::try_from(&raw[..]).ok())
1114 else {
1115 return false;
1116 };
1117 let read = |id: &ObjectId, want: ObjType| match git.read(id.as_bytes()) {
1118 Ok(Some((kind, bytes))) if kind == want => Some(bytes),
1119 _ => None,
1120 };
1121 let Some(commit) = read(&tip, ObjType::Commit).and_then(|bytes| object::commit(&bytes, hash).ok()) else {
1122 return false;
1123 };
1124 let Some(tree) = read(&commit.tree, ObjType::Tree) else {
1125 return false;
1126 };
1127 let Ok(Some(entry)) = object::tree_entry_named(&tree, hash, b".gitattributes") else {
1128 return false;
1129 };
1130 read(&entry.oid, ObjType::Blob).is_some_and(|attributes| {
1131 String::from_utf8_lossy(&attributes)
1132 .lines()
1133 .any(|line| !line.trim_start().starts_with('#') && line.contains("filter=lfs"))
1134 })
1135}
1136
1137/// **The fetch and the apply**, into a repository that exists. Answers what
1138/// the upstream advertised, for [`after_landing`].
1139#[allow(clippy::too_many_arguments)]
1140fn fetch_into(
1141 stores: &Arc<ServerStores>,
1142 row: &gunnar_core::model::Repo,
1143 order: &MirrorOrder,
1144 wire: Wire,
1145 credential: &Credential,
1146 auth: Option<String>,
1147 ceiling: Option<gunnar_wire::PackCeiling>,
1148 report: &Reporter<Mirror>,
1149 watching: Option<&dyn git_storage_trait::PackProgress>,
1150) -> Result<Vec<gunnar_wire::upload_pack::AdvertisedRef>, String> {
1151 let row = row.clone();
1152 let store = stores
1153 .repo_store(&row)
1154 .map_err(|err| format!("opening {} to mirror into it: {err:#}", order.path))?;
1155 let git = store.serve();
1156 let hash_kind = store.hash_kind();
1157
1158 // Opened BEFORE the ingest, and that ordering is load-bearing for the same
1159 // reason it is on the push path: whether this repository has a protection
1160 // rule at all is what decides if the fetch pays for an object-set
1161 // enumeration.
1162 let protection = Protection::open(stores, &row, store.git_dir())?;
1163
1164 let extra_haves = already_here(stores, git, store.git_dir(), order, wire, credential, auth.clone(), report)?;
1165 let local = StoreRefs { git, extra_haves };
1166 let progress = QueueProgress {
1167 report,
1168 total: AtomicU64::new(0),
1169 watching,
1170 };
1171 // The ingest a PUSH lands through, unchanged. A fetched pack and a pushed
1172 // pack are the same bytes (`fetch_pack`'s module doc), so this is the same
1173 // storage authority and not a second one.
1174 //
1175 // The object set is asked for exactly when something reads it. A repository
1176 // with a protection rule has a gate that does (`require_signature` walks the
1177 // commits this fetch introduced), so the enumeration is `Required` and an
1178 // engine that cannot produce one must refuse rather than let the gate decide
1179 // on nothing. With no rule configured nobody reads it, and `NotNeeded` keeps
1180 // the fetch from waiting on index work no answer depends on.
1181 //
1182 // The last argument is the numerator's whole route: the ingest hands it to
1183 // the engine, the engine hands it to the pack walk, and the walk publishes
1184 // once per object into `progress` above — which is the same value the wire
1185 // is publishing the denominator and the byte count into. One sink, three
1186 // publishers, no second decoder anywhere on the path.
1187 let ingest = crate::transport::PushIngest::new(
1188 store.git_dir().to_path_buf(),
1189 hash_kind,
1190 stores.engine(),
1191 if protection.is_some() {
1192 crate::storage::ObjectSet::Required
1193 } else {
1194 crate::storage::ObjectSet::NotNeeded
1195 },
1196 Some(&progress),
1197 // A fetched pack is a transaction exactly as a pushed one is: framed before it lands.
1198 Some(crate::transport::PackFrame::of(stores, &row)),
1199 )
1200 // …and held to the data set's room the way a push is (cov-13).
1201 .with_space(stores.space_ledger());
1202
1203 let mut fetch =
1204 FetchPack::new(hash_kind, &fetch_pack::mirror, &local).with_progress(&progress);
1205 if let Some(ceiling) = ceiling {
1206 fetch = fetch.with_ceiling(ceiling);
1207 }
1208
1209 let outcome = match wire {
1210 Wire::Http => {
1211 let (mut reader, mut writer) = open_http(&order.upstream, auth)?;
1212 fetch.fetch(&ingest, &mut reader, &mut writer)
1213 }
1214 Wire::Ssh => dial_ssh(
1215 &order.upstream,
1216 credential,
1217 &stores.root().join(MIRROR_KNOWN_HOSTS),
1218 strict_host_keys(stores.root()),
1219 |reader, writer| fetch.fetch(&ingest, reader, writer),
1220 )?,
1221 }
1222 .map_err(|err| {
1223 // Cut at the namespace's ceiling: the plan's refusal, as a field.
1224 if matches!(err, gunnar_wire::Error::OverCeiling(_)) {
1225 report.report(|r| r.refused = crate::mirror_queue::Refusal::PackCap);
1226 }
1227 format!(
1228 "fetching {} from {}: {err}",
1229 order.path,
1230 scrub(&order.upstream)
1231 )
1232 })?;
1233
1234 // **Index the fetched pack ONCE, here, before a single read by oid**
1235 // (import-big, 2026-10-26) — the line `receive_pack::apply` has carried
1236 // since 2026-08-30 and this path never got.
1237 //
1238 // The znippy engine stores a pack's bytes before its oids exist and indexes
1239 // them behind the ack. `apply` reads by oid (the branch-protection gate, and
1240 // `put_refs_cas`, whose dangling-target check asks `has` for every new tip),
1241 // so without this its first read fell into the engine's read-time fallback:
1242 // a whole-pack resolve on this thread beside the indexer resolving the same
1243 // bytes. MEASURED on oden 2026-10-26, torvalds/linux from GitHub (6.60 GB,
1244 // 11 986 735 objects): the import landed only after the two resolvers had
1245 // driven the process to a 19.4 GB peak — past a 16 GB Mini, which is where
1246 // the owner's import of it died with nothing to say so.
1247 //
1248 // A refusal is the backend's own sentence (its memory watchdog gives up by
1249 // name, with what was available and what was needed): the pack is stored
1250 // and could not be indexed on this box, which is "this repository does not
1251 // fit here", never a sync that waits for ever or reads around it.
1252 if outcome.receipt.is_some() {
1253 index_landed(git, store.git_dir(), stores.engine(), report).map_err(|err| {
1254 format!(
1255 "{} was fetched and stored, and this server could not index it: {err}. Nothing \
1256 was moved. The repository is larger than this server can take in; a server \
1257 with more memory imports it.",
1258 order.path
1259 )
1260 })?;
1261 }
1262 apply(git, order, &outcome, &progress, protection.as_ref())?;
1263 Ok(outcome.advertised)
1264}
1265
1266/// **`GitServe::index_now`, with the index's own progress on the job.**
1267///
1268/// For a kernel-sized pack the index is minutes after the last byte arrived,
1269/// and a job that sat at `N/N` for all of them would read as stalled. So while
1270/// it runs, the engine's index row for this repository (`Engine::index_progress`,
1271/// the same row the dashboard reads) is published as the job's progress: the
1272/// objects indexed of the pack's total. When it returns the pair is the walk's
1273/// again (`finish`).
1274fn index_landed(
1275 git: &dyn gunnar_store::git_storage_trait::GitServe,
1276 git_dir: &std::path::Path,
1277 engine: crate::storage::Engine,
1278 report: &Reporter<Mirror>,
1279) -> Result<usize, String> {
1280 let done = std::sync::atomic::AtomicBool::new(false);
1281 std::thread::scope(|scope| {
1282 // A32-14, stays a scoped thread: it reads the index row for the length of one
1283 // `index_now` on the mirror job's own lane and is joined before this returns.
1284 scope.spawn(|| {
1285 while !done.load(Ordering::SeqCst) {
1286 if let Some((_, row)) = engine
1287 .index_progress()
1288 .into_iter()
1289 .find(|(dir, row)| dir == git_dir && row.active && row.objects_total > 0)
1290 {
1291 report.progress(row.objects_done, row.objects_total);
1292 }
1293 for _ in 0..10 {
1294 if done.load(Ordering::SeqCst) {
1295 return;
1296 }
1297 std::thread::sleep(std::time::Duration::from_millis(50));
1298 }
1299 }
1300 });
1301 let indexed = git.index_now().map_err(|err| format!("{err:#}"));
1302 done.store(true, Ordering::SeqCst);
1303 if let Ok(packs) = &indexed {
1304 tracing::info!(
1305 target: TARGET_GIT,
1306 event = "mirror.sync.indexed",
1307 packs,
1308 "the fetched pack was indexed once, before the references were checked"
1309 );
1310 }
1311 indexed
1312 })
1313}
1314
1315/// **How much a mirror sync into `namespace` may fetch, decided by the front
1316/// doors' own plan gates** ([`ServerStores::install_plan_gates`]).
1317///
1318/// The push path's questions, in its order (`gunnar_ssh::server`'s
1319/// `ask_plan`): the gate's verdict on a `Write`, then — for a write that grows
1320/// data, which a fetch is — `pack_quota` (is the namespace already at its
1321/// cap?) and the TIGHTEST `pack_ceiling` in the chain. The first `Refuse` is
1322/// the answer, verbatim. The ceiling is bound to the namespace's shared pack
1323/// ledger, marked here before any gate measured, so a mirror fetch and a push
1324/// landing at the same moment share the room instead of each spending it.
1325///
1326/// `Ok(None)` is "nothing caps it": a chain with no gate that answers, which is
1327/// the community appliance's ordinary state.
1328///
1329/// A refusal comes back with WHICH question refused it — the plan's verdict
1330/// on a write ([`crate::mirror_queue::Refusal::Plan`]) or the pack room
1331/// ([`crate::mirror_queue::Refusal::PackCap`]) — so the job can say so as a
1332/// field (`MirrorJob.refused_reason`).
1333pub(crate) fn room_for(
1334 gates: &[Arc<dyn gunnar_ssh::PlanGate>],
1335 namespace: &str,
1336) -> Result<Option<gunnar_wire::PackCeiling>, (crate::mirror_queue::Refusal, String)> {
1337 use gunnar_ssh::{PlanAction, PlanVerdict};
1338 if gates.is_empty() {
1339 return Ok(None);
1340 }
1341 // Taken BEFORE any gate measures, for the reason the push path gives:
1342 // only one of the two roundings is a quota that leaks.
1343 let mark = gunnar_lfs::AllowanceLedger::shared_pack().mark(namespace);
1344 let mut tightest: Option<gunnar_ssh::PackCeiling> = None;
1345 for gate in gates {
1346 if let PlanVerdict::Refuse(line) = gate.verdict(namespace, PlanAction::Write) {
1347 return Err((crate::mirror_queue::Refusal::Plan, line));
1348 }
1349 if let PlanVerdict::Refuse(line) = gate.pack_quota(namespace) {
1350 return Err((crate::mirror_queue::Refusal::PackCap, line));
1351 }
1352 if let Some(asked) = gate.pack_ceiling(namespace) {
1353 if tightest.as_ref().is_none_or(|held| asked.bytes < held.bytes) {
1354 tightest = Some(asked);
1355 }
1356 }
1357 }
1358 Ok(tightest.map(|ceiling| gunnar_wire::PackCeiling {
1359 bytes: ceiling.bytes,
1360 refusal: ceiling.refusal,
1361 ledger: Some(Arc::new(crate::transport::PackMark { mark })
1362 as Arc<dyn gunnar_wire::PackLedger>),
1363 }))
1364}
1365
1366/// **Apply the plan, in one transaction, against the values the fetch
1367/// reasoned about.**
1368///
1369/// See the module doc for why this is `put_refs_cas` and why a rejection is a
1370/// failure rather than a skip.
1371fn apply(
1372 git: &dyn gunnar_store::git_storage_trait::GitServe,
1373 order: &MirrorOrder,
1374 outcome: &FetchReport,
1375 progress: &QueueProgress<'_>,
1376 protection: Option<&Protection>,
1377) -> Result<(), String> {
1378 let zero = "0".repeat(outcome.hash_kind.len_in_hex());
1379
1380 // Owned first, because `RefCas` borrows its oids and the batch has to
1381 // outlive every one of them.
1382 // (name, old, new, peeled)
1383 type OwnedUpdate = (String, Option<Vec<u8>>, Option<Vec<u8>>, Option<Vec<u8>>);
1384 let mut owned: Vec<OwnedUpdate> = Vec::new();
1385 let mut updates: Vec<(String, String, String)> = Vec::new();
1386
1387 for plan in &outcome.updates {
1388 // **The name came off somebody else's wire, so it goes through the ref
1389 // name rules before it is written.**
1390 //
1391 // `plan.name` is the string an upstream's `ls-refs` answered with,
1392 // carried untouched from `fetch_pack`, and the only filter it passed on
1393 // the way is `fetch_pack::mirror`'s prefix test — which admits
1394 // `refs/heads/../../etc/passwd` exactly as readily as `refs/heads/main`.
1395 // Receive-pack puts every client-supplied name through
1396 // `gunnar_store::refname::validate` at the same point in its own path
1397 // (`receive_pack.rs`'s `funny refname`), and a fetch is the same
1398 // untrusted input arriving through a different door.
1399 //
1400 // It is checked HERE rather than relied on from the storage engine. The
1401 // memory backend does validate in `put_refs_cas`, so a name like this
1402 // would be refused there today — but that is a property of one engine,
1403 // and "a rule one backend enforces by accident is not a rule" is the
1404 // sentence `refname` itself is written under. The engines are swappable
1405 // (`GUNNAR_ENGINE`), and this is the one caller that can guarantee the
1406 // check ran.
1407 //
1408 // Only the names that came from the upstream. The prune half below
1409 // reads its names out of this repository's OWN refs, and a bad name
1410 // already in the store is one a prune should be able to delete rather
1411 // than one it must refuse to touch.
1412 gunnar_store::refname::validate(plan.name.as_bytes()).map_err(|err| {
1413 format!(
1414 "the upstream advertised a reference name gunnar will not write: {err}. Nothing \
1415 was moved. A name is not a path, and this one would not have been a name."
1416 )
1417 })?;
1418 // The peel the remote already sent, rather than a peel computed here.
1419 // `ls-refs` was asked with `peel`, so an annotated tag arrived with the
1420 // commit it resolves to; recomputing it would be an object read per tag
1421 // for an answer already in hand (the tag re-peel, from the other side).
1422 let peeled = outcome
1423 .advertised
1424 .iter()
1425 .find(|advertised| advertised.name == plan.name)
1426 .and_then(|advertised| advertised.peeled)
1427 .map(|oid| oid.as_bytes().to_vec());
1428 updates.push((
1429 plan.name.clone(),
1430 plan.old.map(|old| old.to_string()).unwrap_or_else(|| zero.clone()),
1431 plan.new.to_string(),
1432 ));
1433 owned.push((
1434 plan.name.clone(),
1435 plan.old.map(|old| old.as_bytes().to_vec()),
1436 Some(plan.new.as_bytes().to_vec()),
1437 peeled,
1438 ));
1439 }
1440
1441 if order.prune {
1442 // **What the upstream no longer has.** Only over the same namespace the
1443 // ref plan covers (`refs/heads/*`, `refs/tags/*`): a prune that reached
1444 // wider would delete this box's own `refs/gunnar/*` bookkeeping on the
1445 // grounds that an upstream never advertised it.
1446 let held = GitOps::refs(git)
1447 .map_err(|err| format!("reading {}'s refs to prune them: {err:#}", order.path))?;
1448 for row in held {
1449 if !fetch_pack::mirror(&row.name) {
1450 continue;
1451 }
1452 if outcome
1453 .advertised
1454 .iter()
1455 .any(|advertised| advertised.name == row.name)
1456 {
1457 continue;
1458 }
1459 let Some(old) = row.oid else { continue };
1460 updates.push((row.name.clone(), hex(&old), zero.clone()));
1461 owned.push((row.name, Some(old), None, None));
1462 }
1463 }
1464
1465 if owned.is_empty() {
1466 finish(outcome, progress, Vec::new());
1467 return Ok(());
1468 }
1469
1470 // **The gate, between the plan and the write.** Before this line the batch
1471 // below was the one reference write on the box that no gate had seen; a
1472 // repository with no protection rule configured has no `Protection` at all,
1473 // so a mirror of an unprotected repository pays nothing for this.
1474 if let Some(protection) = protection {
1475 let gated: Vec<gunnar_policy::RefUpdate> = owned
1476 .iter()
1477 .map(|(name, old, new, _)| {
1478 // The null-oid spelling never appears here: `owned` already
1479 // carries absence as `None`, which is what `RefUpdate`'s three
1480 // constructors take. `try_from_bytes` cannot fail on a value the
1481 // store or the wire produced, and an oid this store cannot
1482 // decode is the one `StoreRefs::tips` already refuses by name.
1483 let decode = |raw: &Vec<u8>| {
1484 gix_hash::oid::try_from_bytes(raw)
1485 .map(gix_hash::oid::to_owned)
1486 .map_err(|err| {
1487 format!("{name} holds an oid this server cannot decode: {err}")
1488 })
1489 };
1490 Ok(gunnar_policy::RefUpdate {
1491 name: name.clone(),
1492 old: old.as_ref().map(&decode).transpose()?,
1493 new: new.as_ref().map(&decode).transpose()?,
1494 })
1495 })
1496 .collect::<Result<_, String>>()?;
1497 let new_objects = match outcome.receipt.as_ref() {
1498 // Nothing was fetched, so nothing was introduced. That is a measured
1499 // empty set and not an unasked question — see `FetchReport::receipt`.
1500 None => Ok(Vec::new()),
1501 Some(receipt) => receipt.new_objects.clone(),
1502 };
1503 protection.check(git, &gated, new_objects.as_deref().map_err(String::as_str))?;
1504 }
1505
1506 let batch: Vec<RefCas<'_>> = owned
1507 .iter()
1508 .map(|(name, old, new, peeled)| RefCas {
1509 name: name.clone(),
1510 old: old.as_deref(),
1511 new: new.as_deref(),
1512 peeled: peeled.clone(),
1513 })
1514 .collect();
1515
1516 // ONE call. Every edit or none — a loop with a break leaves what it already
1517 // applied applied, and a half-mirrored repository is a state no upstream
1518 // ever had.
1519 GitOps::put_refs_cas(git, &batch).map_err(|err| {
1520 format!(
1521 "the fetch succeeded and the reference update did not: {err:#}. Nothing was moved. \
1522 The usual cause is a push that landed on {} while the fetch was running; the \
1523 expected values came from the snapshot the fetch negotiated against, and \
1524 overwriting a concurrent push is the one thing a mirror must never do. Sync again.",
1525 order.path
1526 )
1527 })?;
1528
1529 tracing::info!(
1530 target: TARGET_GIT,
1531 event = "mirror.sync.applied",
1532 repo = %order.path,
1533 refs = updates.len(),
1534 objects = outcome.objects_done,
1535 bytes = outcome.pack_bytes,
1536 "a mirror sync moved this repository's references"
1537 );
1538
1539 finish(outcome, progress, updates);
1540 Ok(())
1541}
1542
1543/// **Close the job's numbers out**, on both paths out of [`apply`].
1544///
1545/// One function and not two copies, because the pair below is the whole of what
1546/// a console reads when a sync ends and the two paths differ only in whether
1547/// any reference moved.
1548///
1549/// The `progress` call is the second half of the pair `FetchProgress` opened.
1550/// `objects_total` published `(0, N)` the instant the packfile header arrived;
1551/// **this publishes `(N, N)`**, and without it a finished job sits for ever at
1552/// nought per cent of a total it knows — which is worse than no progress bar,
1553/// because it reads as a transfer that stalled. `fetch_pack` has already
1554/// refused the transfer if the declared count and the indexed count disagree,
1555/// so `objects_done` is the same `N` and this cannot report a completion the
1556/// store did not make.
1557///
1558/// A fetch that wanted nothing — the ordinary answer for a poll of an idle
1559/// upstream — has no header, no `N`, and therefore **no progress at all**: the
1560/// pair stays `None`, and a caller reads `has_progress = false` rather than a
1561/// fabricated `0/0`.
1562fn finish(
1563 outcome: &FetchReport,
1564 progress: &QueueProgress<'_>,
1565 updates: Vec<(String, String, String)>,
1566) {
1567 if outcome.receipt.is_some() {
1568 let total = u64::from(outcome.objects_total);
1569 progress.publish(u64::from(outcome.objects_done), total);
1570 }
1571 progress.report.report(|mirror| {
1572 mirror.objects = u64::from(outcome.objects_done);
1573 mirror.bytes = outcome.pack_bytes;
1574 mirror.updates = updates;
1575 });
1576}
1577
1578/// Lower-case hex, the spelling `gunnar.v1.RefUpdate` uses.
1579fn hex(raw: &[u8]) -> String {
1580 let mut out = String::with_capacity(raw.len() * 2);
1581 for byte in raw {
1582 use std::fmt::Write as _;
1583 let _ = write!(out, "{byte:02x}");
1584 }
1585 out
1586}
1587
1588/// `base64(user:secret)`, the Basic-auth pair.
1589///
1590/// Written out rather than pulled in: the whole alphabet is 64 characters and
1591/// the encoder is nine lines, against a dependency that would be in the
1592/// appliance's graph for this one call. It is not a general base64 — it does
1593/// exactly the one thing, and `mirror_http`'s tests assert on the bytes it
1594/// produces.
1595fn base64_basic(user: &str, secret: &str) -> String {
1596 const ALPHABET: &[u8; 64] =
1597 b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
1598 let raw = format!("{user}:{secret}");
1599 let raw = raw.as_bytes();
1600 let mut out = String::with_capacity(raw.len().div_ceil(3) * 4);
1601 for chunk in raw.chunks(3) {
1602 let b = [
1603 chunk[0],
1604 chunk.get(1).copied().unwrap_or(0),
1605 chunk.get(2).copied().unwrap_or(0),
1606 ];
1607 let packed = (u32::from(b[0]) << 16) | (u32::from(b[1]) << 8) | u32::from(b[2]);
1608 out.push(ALPHABET[(packed >> 18) as usize & 63] as char);
1609 out.push(ALPHABET[(packed >> 12) as usize & 63] as char);
1610 out.push(if chunk.len() > 1 {
1611 ALPHABET[(packed >> 6) as usize & 63] as char
1612 } else {
1613 '='
1614 });
1615 out.push(if chunk.len() > 2 {
1616 ALPHABET[packed as usize & 63] as char
1617 } else {
1618 '='
1619 });
1620 }
1621 out
1622}
1623
1624// ── the ssh half ─────────────────────────────────────────────────────────────
1625
1626/// **Dial `git-upload-pack` over SSH and hand the channel's two halves to
1627/// `run`.**
1628///
1629/// `gunnar_client::connect_upload_pack` is the fetch-side twin of the
1630/// `connect_receive_pack` replication already pushes with, over
1631/// `nornir-git`'s russh transport. There is no second dialler here and there
1632/// must not be: that transport owns the connect, the publickey exchange, the
1633/// `known_hosts` policy, the blocking bridge and — load-bearing — the `Drop`
1634/// that tears the channel down *inside* the driving runtime.
1635///
1636/// # The key is a file for the length of one dial, and then it is not
1637///
1638/// `nornir_git::ssh::SshAuth` authenticates from a **path**. gunnar's mirror
1639/// credential is sealed bytes in the control database, because a private key
1640/// sitting in the clear on `/data` is the thing that store exists to prevent.
1641/// The two are bridged by a [`tempfile::NamedTempFile`] created `0600` and
1642/// dropped — and therefore unlinked — before this function returns, on every
1643/// path out of it including a panic.
1644///
1645/// The alternative was an in-memory key API on `nornir-git`, which is the right
1646/// long-term shape, and it is a change to a crate outside this repository. It
1647/// is named here rather than left as a silent compromise.
1648#[cfg(feature = "replication")]
1649fn dial_ssh<T>(
1650 url: &str,
1651 credential: &Credential,
1652 known_hosts: &std::path::Path,
1653 strict: bool,
1654 run: impl FnOnce(&mut dyn BufRead, &mut dyn Write) -> T,
1655) -> Result<T, String> {
1656 use gunnar_client::ssh::{HostKeyPolicy, SshOptions};
1657 use std::io::BufReader;
1658
1659 let mut options = SshOptions::new();
1660 // A server has no ssh-agent and no interactive user. Offering the agent
1661 // would be offering identities nobody configured for this mirror — the
1662 // silent substitution `gunnar_client::resolve_identity` exists to forbid.
1663 options.use_agent = false;
1664
1665 // Held for the whole of `run`: dropping it unlinks the file, and the dial
1666 // reads it during `connect_upload_pack`.
1667 let key_file = match credential {
1668 Credential::SshKey(bytes) => {
1669 let mut file = tempfile::Builder::new()
1670 .prefix("gunnar-mirror-key-")
1671 .tempfile()
1672 .map_err(|err| format!("no room to materialise the mirror's ssh key: {err}"))?;
1673 #[cfg(unix)]
1674 {
1675 use std::os::unix::fs::PermissionsExt as _;
1676 std::fs::set_permissions(file.path(), std::fs::Permissions::from_mode(0o600))
1677 .map_err(|err| {
1678 format!("the mirror's ssh key could not be made private: {err}")
1679 })?;
1680 }
1681 file.write_all(bytes)
1682 .and_then(|()| file.flush())
1683 .map_err(|err| format!("writing the mirror's ssh key: {err}"))?;
1684 options.key_path = Some(file.path().to_path_buf());
1685 Some(file)
1686 }
1687 // No key named means the ladder, which on a server finds nothing; that
1688 // is the honest behaviour for a public upstream over ssh and it is what
1689 // `gunnar-client` warns about at start-up.
1690 _ => None,
1691 };
1692 // Trust-on-first-use against a `known_hosts` under the data root is what
1693 // replication does when an operator has not said otherwise, and a
1694 // mirror is dialling the same kind of box. See [`MIRROR_KNOWN_HOSTS`] for
1695 // why the file is NOT `~/.ssh/known_hosts`.
1696 //
1697 // **Unless the box says otherwise**, which it could not before: the policy
1698 // was this literal and nothing else, so a first sync trusted whatever
1699 // answered on the upstream's address, on a path that is about to offer it a
1700 // private key. See [`MIRROR_STRICT_HOST_KEY`].
1701 options.host_key = if strict {
1702 HostKeyPolicy::StrictKnownHosts
1703 } else {
1704 HostKeyPolicy::TofuKnownHosts
1705 };
1706 options.known_hosts = Some(known_hosts.to_path_buf());
1707
1708 let location = nornir_git::ssh::parse_ssh_url(url)
1709 .map_err(|err| format!("{} is not an ssh url: {err}", scrub(url)))?;
1710 let mut channel = gunnar_client::connect_upload_pack(&location, &options).map_err(|err| {
1711 // `{:#}` over an `anyhow::Error`, and not `{err}`: `gunnar_client::Error`
1712 // is a thiserror enum whose `Ssh` variant Displays as the bare word
1713 // "ssh" and carries the real sentence — which host key, which policy —
1714 // as its `source`. A FAILED job reading `opening git-upload-pack on …:
1715 // ssh` tells an operator nothing they can act on, and the reason was
1716 // one formatting flag away the whole time.
1717 format!(
1718 "opening git-upload-pack on {}: {:#}",
1719 scrub(url),
1720 anyhow::Error::new(err)
1721 )
1722 })?;
1723 let out = {
1724 let (reader, writer) = channel.io();
1725 let mut reader = BufReader::new(reader);
1726 run(&mut reader, writer)
1727 };
1728 drop(key_file);
1729 Ok(out)
1730}
1731
1732/// **What a build with no ssh dialler says**, rather than what it does.
1733///
1734/// `replication` is the feature that carries `gunnar-client`'s wire half and
1735/// `nornir-git`'s russh transport; without it there is no dialler in the
1736/// binary. A mirror registered against an `ssh://` remote on such a build is a
1737/// registration that can never run, and it says so at the one place it can be
1738/// acted on — a FAILED job naming the flag — rather than failing as a
1739/// connection error that sends an operator to look at the network.
1740///
1741/// The smart-HTTP half is behind its own feature, `mirror-http` (A32-18), which every
1742/// appliance row names: [`NO_MIRROR_HTTP`] is its refusal.
1743#[cfg(not(feature = "replication"))]
1744fn dial_ssh<T>(
1745 url: &str,
1746 _credential: &Credential,
1747 _known_hosts: &std::path::Path,
1748 _strict: bool,
1749 _run: impl FnOnce(&mut dyn BufRead, &mut dyn Write) -> T,
1750) -> Result<T, String> {
1751 Err(format!(
1752 "{} is an ssh remote and this build carries no ssh dialler: it was compiled without \
1753 `--features replication`, which is what links `gunnar-client`'s russh transport. \
1754 Mirror from the upstream's `https://` URL, or use a build that has the feature.",
1755 scrub(url)
1756 ))
1757}
1758
1759/// **Take any userinfo out of a URL before it is shown to anybody.**
1760///
1761/// `https://user:token@host/path` is a legal way to write a remote and an
1762/// operator may well have written one. Everything between `//` and the last
1763/// `@` of the authority is replaced, so a token that arrived in the URL cannot
1764/// leave through an error string.
1765///
1766/// Not a parser: this runs in an error path, and a URL this cannot make sense
1767/// of is returned with the whole authority elided rather than passed through.
1768pub(crate) fn scrub(url: &str) -> String {
1769 let Some((scheme, rest)) = url.split_once("://") else {
1770 return url.to_owned();
1771 };
1772 let (authority, path) = match rest.find('/') {
1773 Some(cut) => (&rest[..cut], &rest[cut..]),
1774 None => (rest, ""),
1775 };
1776 match authority.rfind('@') {
1777 Some(cut) => format!("{scheme}://<redacted>@{}{path}", &authority[cut + 1..]),
1778 None => url.to_owned(),
1779 }
1780}
1781
1782/// **An `https://` remote's two halves** ([`crate::mirror_http`], feature `mirror-http`).
1783#[cfg(feature = "mirror-http")]
1784fn open_http(url: &str, auth: Option<String>) -> Result<(crate::mirror_http::HttpRead, crate::mirror_http::HttpWrite), String> {
1785 crate::mirror_http::open(url, auth)
1786}
1787
1788/// The refusal an `https://` remote gets in a build without `mirror-http` (A32-18): the
1789/// airgap default has no HTTP client, and says so where it can be acted on.
1790pub const NO_MIRROR_HTTP: &str = "this build carries no HTTP client: it was compiled without gunnar-server's `mirror-http` feature (the appliance rows name it)";
1791
1792#[cfg(not(feature = "mirror-http"))]
1793fn open_http(url: &str, _auth: Option<String>) -> Result<(std::io::Empty, std::io::Sink), String> {
1794 Err(format!("{} is an https remote and {NO_MIRROR_HTTP}. Mirror from the upstream's ssh:// URL, or use a build that has the feature.", scrub(url)))
1795}
1796
1797#[cfg(test)]
1798mod tests {
1799 use super::*;
1800
1801 /// **The userinfo scrub is the last thing between a token in a URL and the
1802 /// log.** Asserted on the bytes, because "it does not appear" is the whole
1803 /// property.
1804 /// ★ A32-18: the airgap default carries no HTTP client, so an `https://` remote is refused
1805 /// by name (and its token never shown), never dialled.
1806 #[cfg(not(feature = "mirror-http"))]
1807 #[test]
1808 fn an_https_remote_is_refused_by_name_in_a_build_without_mirror_http() {
1809 let e = open_http("https://user:tok3n@forge.mock.invalid/ns/repo.git", Some("Bearer tok3n".into())).err().expect("refused");
1810 assert!(e.contains(NO_MIRROR_HTTP) && e.contains("mirror-http"), "{e}");
1811 assert!(!e.contains("tok3n"), "{e}");
1812 }
1813
1814 #[test]
1815 fn a_url_with_userinfo_never_shows_it() {
1816 let scrubbed = scrub("https://alice:ghp_SUPERSECRET@example.test/ns/repo.git");
1817 assert!(
1818 !scrubbed.contains("ghp_SUPERSECRET"),
1819 "the token survived the scrub: {scrubbed}"
1820 );
1821 assert!(
1822 !scrubbed.contains("alice"),
1823 "the whole userinfo goes, not just the password half: {scrubbed}"
1824 );
1825 assert_eq!(scrubbed, "https://<redacted>@example.test/ns/repo.git");
1826 }
1827
1828 /// A URL with no userinfo is left exactly as the operator wrote it — an
1829 /// error that mangled the ordinary case would be worse than no scrub.
1830 #[test]
1831 fn a_plain_url_is_untouched() {
1832 let url = "https://example.test/ns/repo.git";
1833 assert_eq!(scrub(url), url);
1834 assert_eq!(scrub("ssh://git@example.test/ns/repo.git"), "ssh://<redacted>@example.test/ns/repo.git");
1835 }
1836
1837
1838 /// **Both transports are recognised from the URL alone**, and the scp-like
1839 /// form is ssh — the shape every forge prints on its clone button.
1840 #[test]
1841 fn a_remote_url_names_its_transport() {
1842 assert_eq!(classify("ssh://git@host/ns/repo.git"), Ok(Wire::Ssh));
1843 assert_eq!(classify("git+ssh://git@host/ns/repo.git"), Ok(Wire::Ssh));
1844 assert_eq!(classify("git@host:ns/repo.git"), Ok(Wire::Ssh));
1845 assert_eq!(classify("https://host/ns/repo.git"), Ok(Wire::Http));
1846 assert_eq!(classify("http://127.0.0.1:9418/ns/repo.git"), Ok(Wire::Http));
1847 }
1848
1849 /// A remote gunnar cannot dial is refused when it is REGISTERED, not on the
1850 /// first sync — and the refusal names the two forms that work.
1851 #[test]
1852 fn a_remote_that_is_neither_is_refused_by_name() {
1853 let err = classify("/srv/git/ns/repo.git").expect_err("a local path is not a remote");
1854 assert!(err.contains("ssh://") && err.contains("https://"), "{err}");
1855 let err = classify("git://host/ns/repo.git").expect_err("the dumb git protocol");
1856 assert!(err.contains("scheme gunnar cannot mirror from"), "{err}");
1857 }
1858
1859 /// **The mismatch is refused by name, in both directions.**
1860 ///
1861 /// The failure being prevented is not an error — it is a 401 (or an SSH
1862 /// authentication failure) against an operator who can see that a
1863 /// credential IS stored, and who will therefore rotate a key that was never
1864 /// the problem.
1865 #[test]
1866 fn a_credential_of_the_wrong_kind_is_refused_by_name() {
1867 let key = Credential::SshKey(b"-----BEGIN OPENSSH PRIVATE KEY-----".to_vec());
1868 let token = Credential::Token {
1869 username: String::new(),
1870 secret: "ghp_SECRET".to_owned(),
1871 };
1872
1873 let err = check_credential(Wire::Http, &key).expect_err("a key cannot ride on http");
1874 assert!(err.contains("smart http") && err.contains("ssh private key"), "{err}");
1875 assert!(!err.contains("BEGIN OPENSSH"), "the refusal quoted the key: {err}");
1876
1877 let err = check_credential(Wire::Ssh, &token).expect_err("a token cannot ride on ssh");
1878 assert!(err.contains("ssh") && err.contains("a token"), "{err}");
1879 assert!(!err.contains("ghp_SECRET"), "the refusal quoted the token: {err}");
1880
1881 // …and the matching pairs, plus the public case, are all fine.
1882 check_credential(Wire::Ssh, &key).expect("a key over ssh");
1883 check_credential(Wire::Http, &token).expect("a token over http");
1884 check_credential(Wire::Ssh, &Credential::None).expect("a public ssh upstream");
1885 check_credential(Wire::Http, &Credential::None).expect("a public http upstream");
1886 }
1887
1888 /// **The authority, out of each of the three shapes** — and the default
1889 /// port with it, because the port is half of what gets dialled.
1890 #[test]
1891 fn a_remote_url_names_the_machine_it_would_dial() {
1892 for (url, want) in [
1893 ("https://host/ns/repo.git", ("host".to_owned(), 443)),
1894 ("http://host/ns/repo.git", ("host".to_owned(), 80)),
1895 ("http://host:9418/ns/repo.git", ("host".to_owned(), 9418)),
1896 ("ssh://git@host/ns/repo.git", ("host".to_owned(), 22)),
1897 ("ssh://git@host:2222/ns/repo.git", ("host".to_owned(), 2222)),
1898 ("git@host:ns/repo.git", ("host".to_owned(), 22)),
1899 // The userinfo is not the host, and a `@` inside it must not become
1900 // one either.
1901 ("https://user:pass@host/ns/repo.git", ("host".to_owned(), 443)),
1902 // An IPv6 literal is all colons; the brackets are what separates the
1903 // address from the port, and splitting on the first colon would
1904 // yield `[`.
1905 ("ssh://git@[::1]:2222/ns/repo.git", ("::1".to_owned(), 2222)),
1906 (
1907 "https://[2606:4700::1111]/ns/repo.git",
1908 ("2606:4700::1111".to_owned(), 443),
1909 ),
1910 ("git@[::1]:ns/repo.git", ("::1".to_owned(), 22)),
1911 ] {
1912 assert_eq!(authority(url), Ok(want.clone()), "{url}");
1913 }
1914 assert!(authority("https:///ns/repo.git").is_err(), "no host at all");
1915 }
1916
1917 /// **Every range a caller must not be able to point this server at**, and
1918 /// the two that must still work.
1919 ///
1920 /// Asserted on [`address_verdict`] rather than on [`check_destination`]
1921 /// because the addresses are handed in: no resolver runs, so this is the
1922 /// same answer on a laptop, on a build box with a corporate DNS and in a
1923 /// container with none.
1924 #[test]
1925 fn a_mirror_refuses_every_address_only_the_server_can_reach() {
1926 for addr in [
1927 "127.0.0.1",
1928 "0.0.0.0",
1929 "10.0.0.5",
1930 "172.16.0.1",
1931 "192.168.1.1",
1932 // The one the finding names: cloud instance metadata.
1933 "169.254.169.254",
1934 "100.64.0.1",
1935 "192.0.0.1",
1936 "198.18.0.1",
1937 "240.0.0.1",
1938 "255.255.255.255",
1939 "224.0.0.1",
1940 "::1",
1941 "::",
1942 "fc00::1",
1943 "fe80::1",
1944 "ff02::1",
1945 // …written the long way. Without the mapped-address arm this one
1946 // walks past every v6 test and is allowed.
1947 "::ffff:127.0.0.1",
1948 "::ffff:169.254.169.254",
1949 ] {
1950 let ip: std::net::IpAddr = addr.parse().expect("a literal");
1951 let err = address_verdict("upstream.invalid", &[ip])
1952 .expect_err("{addr} must not be dialled on a caller's say-so");
1953 assert!(
1954 err.contains(addr) || err.contains(&ip.to_string()),
1955 "{addr}: {err}"
1956 );
1957 assert!(err.contains(MIRROR_ALLOW_PRIVATE), "{addr}: {err}");
1958 }
1959
1960 for addr in ["93.184.216.34", "8.8.8.8", "2606:4700:4700::1111"] {
1961 let ip: std::net::IpAddr = addr.parse().expect("a literal");
1962 address_verdict("upstream.invalid", &[ip])
1963 .unwrap_or_else(|err| panic!("{addr} is an ordinary public address: {err}"));
1964 }
1965 }
1966
1967 /// **One bad answer out of several is a refusal.** A name that resolves to a
1968 /// routable address and a loopback address is the rebinding shape, and a
1969 /// check that took the first answer, or the best one, would pass it.
1970 #[test]
1971 fn one_private_answer_among_several_refuses_the_whole_name() {
1972 let addrs: Vec<std::net::IpAddr> = ["93.184.216.34", "127.0.0.1"]
1973 .iter()
1974 .map(|a| a.parse().expect("a literal"))
1975 .collect();
1976 let err = address_verdict("both.invalid", &addrs).expect_err("one of them is loopback");
1977 assert!(err.contains("127.0.0.1"), "{err}");
1978
1979 // And nothing at all is not "fine": a name with no address is refused
1980 // rather than allowed through to a dial that would fail anyway.
1981 assert!(address_verdict("nowhere.invalid", &[]).is_err());
1982 }
1983
1984 /// The Basic pair is encoded the way every HTTP client encodes it. Checked
1985 /// against a value anybody can verify by hand rather than against this
1986 /// function's own output.
1987 #[test]
1988 fn basic_auth_is_ordinary_base64() {
1989 assert_eq!(base64_basic("Aladdin", "open sesame"), "QWxhZGRpbjpvcGVuIHNlc2FtZQ==");
1990 assert_eq!(base64_basic("a", "b"), "YTpi");
1991 assert_eq!(base64_basic("ab", "b"), "YWI6Yg==");
1992 }
1993}