Skip to main content

gunnar_server/
mirror_fetch.rs

1//! **What `Mirror.Sync` actually does**: one fetch from an upstream, into this
2//! box's own storage engine, and one compare-and-swap that moves the
3//! references.
4//!
5//! # There is one client, and two ways to reach a remote
6//!
7//! [`gunnar_wire::fetch_pack`] is the whole protocol — the advertisement, the
8//! `ls-refs`, the want/have negotiation, the sideband demultiplex, the packfile
9//! streamed into a [`PackIngest`](gunnar_wire::PackIngest). It takes a
10//! `&mut dyn BufRead` and a `&mut dyn Write` and cannot tell what is behind
11//! them, which is the seam this module fills twice and only twice:
12//!
13//! | remote | the pair | who built it |
14//! |---|---|---|
15//! | `ssh://`, `git+ssh://`, `git@host:path` | a russh channel's two halves | `nornir-git`, through `gunnar_client::connect_upload_pack` — the **same dialler** replication pushes over |
16//! | `https://`, `http://` | a buffered request/response adapter | [`crate::mirror_http`] |
17//!
18//! Neither is a second protocol implementation, and that is the point. A bug in
19//! negotiation is one bug, in one file, in `gunnar-wire`.
20//!
21//! # This module writes the references, and `fetch_pack` deliberately does not
22//!
23//! `fetch_pack`'s module doc states the rule: it returns a **plan**
24//! (`FetchReport::updates`, each entry a name, the old value and the new one)
25//! and applies nothing, because applying a whole mirror's plan with no gate, no
26//! atomicity across the batch and no way for an operator to see it first is not
27//! something a protocol crate should be able to do.
28//!
29//! So the apply is here, and it is **one [`GitOps::put_refs_cas`]** with the
30//! plan's own `old` as every expectation. That is not tidiness:
31//!
32//! * A mirror is not the only writer. `git push` reaches the same repository
33//!   over SSH and over smart HTTP, and a fetch that read a tip, spent four
34//!   minutes on the network and then wrote unconditionally would **silently
35//!   destroy** whatever landed in between. The `old` in the plan is read from
36//!   the same `GitOps::refs()` snapshot the `have` list came from, so the CAS
37//!   is checking exactly the value the fetch reasoned about.
38//! * A CAS failure is a **FAILED job that names the reference**, never a skip.
39//!   "Synced" about a repository where one branch quietly did not move is the
40//!   answer that makes a mirror untrustworthy, and it reads identically to a
41//!   mirror that worked.
42//! * Every edit or none. A half-applied plan leaves the repository at a state
43//!   no upstream ever had.
44//!
45//! # …and the batch goes through branch protection first
46//!
47//! A mirror moves references, and until 2026-09-11 it was the only writer on
48//! this box that moved them without asking [`BranchProtectionGate`]. That is
49//! not a smaller version of a push: `Mirror.Sync` needs **Write** and
50//! `Policy.SetProtection` needs **Admin**, so a principal who may not configure
51//! a protection rule could register a mirror against the repository it protects
52//! and have the upstream's tips applied over it — a non-fast-forward onto a ref
53//! whose rule says `allow_force_push: false`, applied by the one path that
54//! never looked. See [`Protection`] for what is decided here and what is
55//! refused as undecidable.
56//!
57//! # The credential is never in this module's output
58//!
59//! The value is fetched from the sealed secret store, held for the length of
60//! one dial, and never formatted. An SSH key is materialised into a
61//! `0600` temporary file because `nornir-git` authenticates from a path and
62//! this tree does not fork a second dialler to change that (one implementation); the file is
63//! unlinked when the fetch returns, on every path including a panic, because
64//! [`tempfile::NamedTempFile`] unlinks on `Drop`. A token becomes an
65//! `Authorization` header value inside [`crate::mirror_http`] and nothing else.
66//! Errors that mention a URL go through [`scrub`] first.
67
68use std::io::{BufRead, Write};
69use std::sync::atomic::{AtomicU64, Ordering};
70use std::sync::Arc;
71
72use gix_hash::ObjectId;
73use gunnar_core::MirrorCredentialKind;
74use gunnar_store::git_storage_trait::{GitOps, RefCas};
75use gunnar_wire::fetch_pack::{self, FetchPack, FetchProgress, FetchReport, LocalRefs};
76
77use crate::job_queue::Reporter;
78use crate::mirror_queue::{Mirror, MirrorOrder};
79use crate::stores::ServerStores;
80
81/// The tracing target a sync's lines carry — the same one every other git
82/// transfer on this box uses, so `RUST_LOG=gunnar::git=info` selects a mirror
83/// as well as a clone.
84const TARGET_GIT: &str = "gunnar::git";
85
86/// **The `known_hosts` a mirror's SSH dial trusts**, under the data root.
87///
88/// Not `~/.ssh/known_hosts`, and this is the same ruling replication's
89/// `REPLICATION_KNOWN_HOSTS` makes for the push direction: a server's identity
90/// is the box, not a home directory. The appliance has no home directory at
91/// all, so the default would be a file that cannot be written and a
92/// trust-on-first-use that silently never remembers — every sync a first
93/// contact, which is the same as no host-key checking with extra steps.
94///
95/// A separate file from replication's, because they are separate rosters: the
96/// standbys this box pushes to and the upstreams it pulls from are different
97/// sets of machines, and a pin acquired for one is not consent for the other.
98pub const MIRROR_KNOWN_HOSTS: &str = "mirror_known_hosts";
99
100/// **The file that makes a mirror's SSH dial refuse an unknown host key**,
101/// under the data root beside [`MIRROR_KNOWN_HOSTS`].
102///
103/// Without it a mirror dials trust-on-first-use: an upstream this box has never
104/// seen is recorded and trusted, and only a *changed* key is refused. That is
105/// the right default for a developer's laptop and a defensible one for a server
106/// being brought up without a provisioning step — it is what replication's own
107/// `strict_host_key` defaults to, and its comment already says that on "is the
108/// right setting for anything that outlives an afternoon". The mirror side had
109/// no way to say on at all: the policy was a literal in [`dial_ssh`].
110///
111/// A file rather than a field on `MirrorRecord`, for the reason
112/// [`MIRROR_KNOWN_HOSTS`] is a file under the data root: the roster of hosts
113/// this box will pull from is the box's, not one repository's, and the pin it
114/// is checked against is already per-box. Create it once, and every mirror on
115/// the box requires a `known_hosts` entry that somebody put there.
116pub const MIRROR_STRICT_HOST_KEY: &str = "mirror_strict_host_key";
117
118/// Does this box refuse an unknown upstream host key? See
119/// [`MIRROR_STRICT_HOST_KEY`].
120///
121/// The presence of the file is read at each dial rather than cached, so an
122/// operator who creates it does not have to restart the server to mean it.
123pub fn strict_host_keys(data_root: &std::path::Path) -> bool {
124    data_root.join(MIRROR_STRICT_HOST_KEY).exists()
125}
126
127/// **The file that lets this box's mirrors dial an address that is not
128/// publicly routable**, under the data root beside [`MIRROR_KNOWN_HOSTS`].
129///
130/// A file and not an environment variable, for the reason `wg.rs` writes out at
131/// length: an env var is process-global, so a rule that can only be exercised
132/// by `set_var` is a rule whose tests flake the moment `cargo test` runs them
133/// on two threads. A file under the data root is per-box, survives a restart,
134/// is visible to an operator looking at the box, and each test gets its own.
135///
136/// It exists because refusing every private destination outright would be a
137/// product regression, not a fix: mirroring from a forge on the operator's own
138/// LAN is an ordinary thing for an appliance to do. What must not be ordinary
139/// is a **low-privileged** caller getting it for free — see
140/// [`check_destination`].
141pub const MIRROR_ALLOW_PRIVATE: &str = "mirror_allow_private_destinations";
142
143/// **The credential a mirror dials with**, in the one shape it exists in
144/// memory.
145///
146/// Deliberately not `Debug`, `Clone` or `Serialize`. The compiler is the only
147/// mechanism that reliably stops a secret being formatted into a log line by
148/// somebody adding a `{:?}` three months from now, so the derive that would
149/// allow it is simply absent.
150pub enum Credential {
151    /// No credential. A public upstream needs none, and a mirror of one works.
152    None,
153    /// An OpenSSH private key, for an `ssh://` upstream.
154    SshKey(Vec<u8>),
155    /// A token, for an `https://` upstream. `username` empty means `Bearer`.
156    Token { username: String, secret: String },
157}
158
159impl Credential {
160    fn kind(&self) -> MirrorCredentialKind {
161        match self {
162            Credential::None => MirrorCredentialKind::None,
163            Credential::SshKey(_) => MirrorCredentialKind::SshKey,
164            Credential::Token { .. } => MirrorCredentialKind::Token,
165        }
166    }
167}
168
169/// **Which transport a remote URL names.**
170///
171/// Decided by the URL and by nothing else — not by which credential happens to
172/// be stored, and not by trying one and falling back. A fallback would turn "I
173/// gave you the wrong kind of key" into "your upstream is down", and those go
174/// to two different people.
175#[derive(Debug, Clone, Copy, PartialEq, Eq)]
176pub enum Wire {
177    /// `ssh://`, `git+ssh://`, or the scp-like `git@host:ns/repo.git`.
178    Ssh,
179    /// `https://` or `http://`.
180    Http,
181}
182
183impl Wire {
184    /// The word an operator sees in a refusal.
185    fn name(self) -> &'static str {
186        match self {
187            Wire::Ssh => "ssh",
188            Wire::Http => "smart http",
189        }
190    }
191}
192
193/// **What transport `url` names, or a refusal that says what gunnar can dial.**
194///
195/// The scp-like form is recognised the way git recognises it: a `:` before any
196/// `/`, with no scheme. A URL that is neither is refused rather than guessed
197/// at — a mirror registered against a path on the local disk would otherwise
198/// look like it was configured and then fail on the first sync with a
199/// connection error.
200pub fn classify(url: &str) -> Result<Wire, String> {
201    let url = url.trim();
202    if url.starts_with("ssh://") || url.starts_with("git+ssh://") {
203        return Ok(Wire::Ssh);
204    }
205    if url.starts_with("https://") || url.starts_with("http://") {
206        return Ok(Wire::Http);
207    }
208    if url.contains("://") {
209        return Err(format!(
210            "{} names a scheme gunnar cannot mirror from. The two transports are `ssh://` \
211             and `https://`",
212            scrub(url)
213        ));
214    }
215    // `git@host:ns/repo.git` — a colon before any slash, and a host in front
216    // of it.
217    match (url.find(':'), url.find('/')) {
218        (Some(colon), slash) if colon > 0 && slash.is_none_or(|slash| colon < slash) => {
219            Ok(Wire::Ssh)
220        }
221        _ => Err(format!(
222            "{url:?} is not a remote gunnar can mirror from. Write it as \
223             `ssh://git@host/ns/repo.git`, `git@host:ns/repo.git` or `https://host/ns/repo.git`"
224        )),
225    }
226}
227
228// ── where a mirror may dial ──────────────────────────────────────────────────
229
230/// **The host and port a mirror URL names**, in the three shapes
231/// [`classify`] admits.
232///
233/// Deliberately not a URL parser: the only question asked of this string is
234/// *which machine would be dialled*, and the answer is the authority. A
235/// userinfo half is dropped — `git@host` names `host` — and an IPv6 literal
236/// keeps its brackets off so it can be parsed as an address.
237pub(crate) fn authority(url: &str) -> Result<(String, u16), String> {
238    let url = url.trim();
239    let (default_port, rest) = if let Some(rest) = url.strip_prefix("ssh://") {
240        (22, rest)
241    } else if let Some(rest) = url.strip_prefix("git+ssh://") {
242        (22, rest)
243    } else if let Some(rest) = url.strip_prefix("https://") {
244        (443, rest)
245    } else if let Some(rest) = url.strip_prefix("http://") {
246        (80, rest)
247    } else {
248        // The scp-like `git@host:ns/repo.git`. `classify` has already decided
249        // this shape is ssh; the colon here separates a PATH, never a port.
250        // The userinfo goes first so a `@` inside it cannot be read as a host,
251        // and a bracketed literal is taken whole — `[::1]:path` is all colons
252        // and splitting on the first one would yield `[`.
253        let rest = url.rsplit('@').next().unwrap_or_default();
254        let host = match rest.strip_prefix('[') {
255            Some(inside) => inside.split(']').next().unwrap_or_default(),
256            None => rest.split(':').next().unwrap_or_default(),
257        };
258        return non_empty(host, url).map(|host| (host, 22));
259    };
260    let authority = rest.split(['/', '?', '#']).next().unwrap_or_default();
261    let authority = authority.rsplit('@').next().unwrap_or_default();
262    // `[::1]:9418` and `[::1]`, before the bare-host split, because an IPv6
263    // literal is all colons.
264    if let Some(rest) = authority.strip_prefix('[') {
265        let Some((host, tail)) = rest.split_once(']') else {
266            return Err(format!(
267                "{} names an unterminated IPv6 literal",
268                scrub(url)
269            ));
270        };
271        let port = match tail.strip_prefix(':') {
272            None => default_port,
273            Some(digits) => digits.parse().map_err(|_| {
274                format!(
275                    "{} names a port that is not a number",
276                    scrub(url)
277                )
278            })?,
279        };
280        return non_empty(host, url).map(|host| (host, port));
281    }
282    match authority.split_once(':') {
283        None => non_empty(authority, url).map(|host| (host, default_port)),
284        Some((host, digits)) => {
285            let port = digits.parse().map_err(|_| {
286                format!(
287                    "{} names a port that is not a number",
288                    scrub(url)
289                )
290            })?;
291            non_empty(host, url).map(|host| (host, port))
292        }
293    }
294}
295
296fn non_empty(host: &str, url: &str) -> Result<String, String> {
297    if host.is_empty() {
298        return Err(format!(
299            "{} names no host to dial",
300            scrub(url)
301        ));
302    }
303    Ok(host.to_owned())
304}
305
306/// **Refuse a destination this server can reach and its caller cannot** — the
307/// pure half, with the addresses handed in.
308///
309/// `Mirror.Add` needs Admin and `Mirror.Sync` needs Write, but *every enrolled
310/// principal is intrinsically Admin in its own namespace*
311/// (`AuthTable::owns_account`), so `Mirror.Add{create_repo}` against
312/// `alice/anything` is available to anybody with a key. Before this check the
313/// only thing standing between that caller and an arbitrary request from the
314/// server's own network position was `remote_url.trim().is_empty()`: the cloud
315/// metadata endpoint at `169.254.169.254`, a control plane on `127.0.0.1`, a
316/// printer on `10.0.0.0/8` — all dialled by the appliance, with the appliance's
317/// routes, and the outcome partly readable back through `Mirror.Status`.
318///
319/// **Every** resolved address has to be routable, not merely the first: a name
320/// that resolves to one public address and one loopback address is the
321/// rebinding shape, and taking the best of the two is how a check like this
322/// gets to be decorative.
323///
324/// The ranges are written out rather than taken from `IpAddr::is_global`, which
325/// is unstable, and each line is the reason it is there.
326pub(crate) fn address_verdict(host: &str, addrs: &[std::net::IpAddr]) -> Result<(), String> {
327    if addrs.is_empty() {
328        return Err(format!("{host} resolves to no address at all"));
329    }
330    for addr in addrs {
331        if let Some(why) = not_routable(*addr) {
332            return Err(format!(
333                "{host} resolves to {addr}, which is {why}. A mirror dials from the server, so a \
334                 destination the caller could not reach on their own is the server's network \
335                 position being borrowed. Create `{MIRROR_ALLOW_PRIVATE}` under this box's data \
336                 root to allow it deliberately."
337            ));
338        }
339    }
340    Ok(())
341}
342
343/// Why `addr` is not somewhere an arbitrary caller should be able to point this
344/// server, or `None` when it is an ordinary public address.
345fn not_routable(addr: std::net::IpAddr) -> Option<&'static str> {
346    use std::net::IpAddr;
347    match addr {
348        IpAddr::V4(v4) => {
349            let [a, b, c, _] = v4.octets();
350            match () {
351                _ if v4.is_unspecified() => Some("the unspecified address"),
352                _ if v4.is_loopback() => Some("loopback — this server itself"),
353                _ if v4.is_private() => Some("a private network (RFC 1918)"),
354                _ if v4.is_link_local() => Some(
355                    "link-local (RFC 3927), the range cloud metadata services answer on",
356                ),
357                _ if v4.is_broadcast() => Some("the broadcast address"),
358                _ if v4.is_multicast() => Some("multicast"),
359                _ if v4.is_documentation() => Some("a documentation range (RFC 5737)"),
360                // 100.64.0.0/10, carrier-grade NAT: the provider's own network,
361                // not the caller's.
362                _ if a == 100 && (64..128).contains(&b) => Some("carrier-grade NAT (RFC 6598)"),
363                // 192.0.0.0/24, IETF protocol assignments.
364                _ if a == 192 && b == 0 && c == 0 => Some("an IETF protocol assignment"),
365                // 198.18.0.0/15, benchmarking.
366                _ if a == 198 && (b == 18 || b == 19) => Some("a benchmarking range (RFC 2544)"),
367                // 240.0.0.0/4, reserved.
368                _ if a >= 240 => Some("a reserved range"),
369                _ => None,
370            }
371        }
372        // A v4 address written the long way is the same address. Checked
373        // first, or `::ffff:127.0.0.1` walks straight past every arm below it.
374        IpAddr::V6(v6) if v6.to_ipv4_mapped().is_some() => {
375            not_routable(IpAddr::V4(v6.to_ipv4_mapped().expect("just matched")))
376        }
377        IpAddr::V6(v6) => {
378            let segments = v6.segments();
379            match () {
380                _ if v6.is_unspecified() => Some("the unspecified address"),
381                _ if v6.is_loopback() => Some("loopback — this server itself"),
382                _ if v6.is_multicast() => Some("multicast"),
383                // fc00::/7, unique local.
384                _ if segments[0] & 0xfe00 == 0xfc00 => Some("a unique-local address (RFC 4193)"),
385                // fe80::/10, link-local.
386                _ if segments[0] & 0xffc0 == 0xfe80 => Some("link-local (RFC 4291)"),
387                // 2001:db8::/32, documentation.
388                _ if segments[0] == 0x2001 && segments[1] == 0x0db8 => {
389                    Some("a documentation range (RFC 3849)")
390                }
391                _ => None,
392            }
393        }
394    }
395}
396
397/// [`address_verdict`] with the resolution and the opt-out file read — the
398/// edge, and the only part of this that touches the world.
399///
400/// # The window this does not close, said rather than glossed
401///
402/// The name is resolved here and again by `ureq` or by the ssh dialler a moment
403/// later, so a resolver that answers differently the second time (DNS
404/// rebinding) is not stopped by this. Closing that needs the dial pinned to the
405/// address that was checked, which is a change inside `ureq`'s and
406/// `nornir-git`'s connect paths rather than in this file. What this does close
407/// is the whole of the *static* case — a URL that simply names an internal
408/// address — which is what `Mirror.Add` accepted unchecked.
409pub fn check_destination(url: &str, data_root: &std::path::Path) -> Result<(), String> {
410    use std::net::ToSocketAddrs as _;
411    if data_root.join(MIRROR_ALLOW_PRIVATE).exists() {
412        return Ok(());
413    }
414    let (host, port) = authority(url)?;
415    // A literal needs no resolver, and asking one for it would let a DNS server
416    // answer a question that has no question in it.
417    if let Ok(addr) = host.parse::<std::net::IpAddr>() {
418        return address_verdict(&host, &[addr]);
419    }
420    let addrs: Vec<std::net::IpAddr> = (host.as_str(), port)
421        .to_socket_addrs()
422        .map_err(|err| format!("{host} could not be resolved: {err}"))?
423        .map(|socket| socket.ip())
424        .collect();
425    address_verdict(&host, &addrs)
426}
427
428/// **Refuse a credential that cannot be offered over the transport the URL
429/// names — by name, before anything is dialled.**
430///
431/// The failure this prevents is the quiet one. An SSH key against an `https://`
432/// remote is not "authentication failed": there is nowhere in an HTTP request
433/// to put a private key, so the request goes out anonymous and the upstream
434/// answers 401 — and the operator, who can see that a credential IS stored,
435/// concludes the key is wrong and rotates it. Forever. The same in reverse: a
436/// token has no place in an SSH publickey exchange, so the dial falls through
437/// to whatever else is offerable and fails on authentication with no mention of
438/// the token at all.
439///
440/// A stored credential of no kind is not a mismatch — a public upstream mirrors
441/// fine with nothing, and that path must keep working.
442pub fn check_credential(wire: Wire, credential: &Credential) -> Result<(), String> {
443    match (wire, credential) {
444        (_, Credential::None) => Ok(()),
445        (Wire::Ssh, Credential::SshKey(_)) => Ok(()),
446        (Wire::Http, Credential::Token { .. }) => Ok(()),
447        (wire, held) => Err(format!(
448            "this mirror's remote is dialled over {}, and the credential stored for it is {}. \
449             {} cannot carry it, so the fetch would go out unauthenticated and be refused as \
450             though no credential were set. Store {} instead, with `Mirror.SetCredential`.",
451            wire.name(),
452            held.kind().name(),
453            wire.name(),
454            match wire {
455                Wire::Ssh => "an ssh private key",
456                Wire::Http => "a token",
457            }
458        )),
459    }
460}
461
462// ── the gate ─────────────────────────────────────────────────────────────────
463
464/// **The repository's branch protection, over the batch a mirror is about to
465/// apply.**
466///
467/// One implementation: this is `gunnar_policy::BranchProtectionGate`, the same
468/// object `receive_pack`'s [`crate::transport::PolicyGate`] evaluates, reading
469/// the same document through the same [`crate::transport::load_policy`]. A
470/// second copy of "what does `allow_force_push` mean" is precisely the pair
471/// that drifts, and the three independent ways past branch protection this
472/// repository has had are all the same shape: a second writer that does what
473/// the gated writer does, without the gate.
474///
475/// # What a mirror can decide, and the one thing it cannot
476///
477/// `allow_create`, `allow_delete` and `allow_force_push` are decidable from the
478/// batch and the objects, which are already in the store when [`apply`] runs.
479/// `require_signature` is decidable too — but only against the set of objects
480/// the fetch introduced, so a repository carrying any protection rule makes the
481/// ingest ask for that set ([`crate::storage::ObjectSet::Required`]) instead of
482/// paying nothing for it, exactly as a push does when a gate is live.
483///
484/// `allowed_principals` is the one that cannot be decided here, and it is
485/// refused rather than skipped. The caller's identity does not cross the
486/// control-plane seam on this verb — `mirror_sync(&self, path)` carries a
487/// repository and nothing about who asked — so there is no principal to compare
488/// against the list. "I could not tell" is not "it is fine", so a rule that
489/// names its principals refuses the whole sync by name, and the operator either
490/// drops the rule or stops mirroring onto the refs it covers.
491struct Protection {
492    gate: gunnar_policy::BranchProtectionGate,
493    keys: gunnar_policy::StaticKeyDirectory,
494    repo: gunnar_policy::RepoId,
495}
496
497impl Protection {
498    /// The rules covering `repo`, or `None` when it has none configured.
499    ///
500    /// Read **before** the fetch, because whether any rule exists is what
501    /// decides if the ingest pays for an object-set enumeration — the same
502    /// ordering, for the same reason, as `receive_pack`'s.
503    ///
504    /// `Err` is a policy document that could not be read, and it REFUSES the
505    /// sync — the same fail-closed rule the push path has (GUN-007). A mirror
506    /// that synced with its gates disabled because the policy did not parse
507    /// would be the bypass GUN-032 closed, back through the side door.
508    fn open(
509        stores: &ServerStores,
510        row: &gunnar_core::Repo,
511        git_dir: &std::path::Path,
512    ) -> std::result::Result<Option<Self>, String> {
513        let config = crate::transport::load_policy(
514            stores,
515            row,
516            &git_dir.join(crate::transport::POLICY_FILE),
517        )
518        .map_err(|why| format!("refusing to mirror into {}: {why}", row.key()))?;
519        if config.protection.is_empty() {
520            return Ok(None);
521        }
522        Ok(Some(Protection {
523            gate: gunnar_policy::BranchProtectionGate::new(config.protection.clone()),
524            keys: crate::transport::key_directory(stores),
525            repo: gunnar_policy::RepoId::new(&row.namespace, &row.name),
526        }))
527    }
528
529    /// **Refuse the batch, or let it through.** `Err` is the refusal a FAILED
530    /// job carries, and nothing has been written when it is returned.
531    ///
532    /// `new_objects` is the fetch's own receipt. An ingest that did not
533    /// enumerate hands back the reason rather than an empty set, and an empty
534    /// set here would be a silent always-allow for `require_signature` — so the
535    /// `Err` arm refuses, quoting why.
536    fn check(
537        &self,
538        git: &dyn gunnar_store::git_storage_trait::GitServe,
539        updates: &[gunnar_policy::RefUpdate],
540        new_objects: std::result::Result<&[ObjectId], &str>,
541    ) -> Result<(), String> {
542        use gunnar_policy::Gate as _;
543
544        // The field with no answer on this path, refused first and by name.
545        for update in updates {
546            for rule in self.gate.rules() {
547                if !rule.pattern.matches_ref(&update.name) {
548                    continue;
549                }
550                if rule.allowed_principals.is_some() {
551                    return Err(format!(
552                        "{} is protected by {}, which names the principals that may move it. A \
553                         mirror sync carries no principal — the control-plane verb takes a \
554                         repository and not a caller — so that rule cannot be satisfied and this \
555                         sync is refused rather than applied unchecked. Nothing was moved.",
556                        update.name, rule.pattern
557                    ));
558                }
559                break;
560            }
561        }
562
563        let new_objects = new_objects.map_err(|reason| {
564            format!(
565                "this repository has branch protection and the fetched pack's object set could \
566                 not be enumerated ({reason}), so the gate could not decide. Nothing was moved."
567            )
568        })?;
569        // Never consulted: every rule that could read it has already refused
570        // above. Named rather than left as a plausible-looking value.
571        let principal = gunnar_policy::Principal::machine("mirror");
572        let ctx = gunnar_policy::PushContext::new(
573            &principal,
574            &self.repo,
575            updates,
576            new_objects,
577            git,
578            &self.keys,
579        );
580        match self.gate.check(&ctx) {
581            Ok(gunnar_policy::Verdict::Allow) => Ok(()),
582            Ok(gunnar_policy::Verdict::Reject(rejection)) => Err(format!(
583                "branch protection refused this mirror's reference batch: {}. Nothing was moved.",
584                rejection.reason
585            )),
586            // A gate that could not decide refuses; it does not allow.
587            Err(err) => Err(format!(
588                "branch protection could not decide this mirror's reference batch: {err}. \
589                 Nothing was moved."
590            )),
591        }
592    }
593}
594
595// ── the local half ───────────────────────────────────────────────────────────
596
597/// **What this repository already holds**, as `fetch_pack`'s have-source.
598///
599/// One `GitOps::refs()` call, one snapshot, taken inside `FetchPack::fetch` so
600/// the `have` list and every `old` in the plan come from the same read — a
601/// reference that moves under the fetch cannot make the plan promise an old
602/// value the store never held.
603///
604/// `HEAD` and the pseudo-refs are not rows in `refs()` at all, and a purely
605/// symbolic row under `refs/` carries no oid and is skipped: it is not a thing
606/// to offer as a `have`, and `fetch_pack`'s own doc says an unborn ref should
607/// simply be absent.
608struct StoreRefs<'a> {
609    git: &'a dyn gunnar_store::git_storage_trait::GitServe,
610    /// Objects this store holds that no reference names yet — an interrupted
611    /// import's ([`already_here`]). Offered as `have`s under names the plan
612    /// never matches, so they change what is sent and nothing that is written.
613    extra_haves: Vec<ObjectId>,
614}
615
616impl LocalRefs for StoreRefs<'_> {
617    fn tips(&self) -> gunnar_wire::Result<Vec<(String, ObjectId)>> {
618        let rows = GitOps::refs(self.git).map_err(|err| {
619            gunnar_wire::Error::Store(gunnar_store::Error::backend(format!(
620                "reading this repository's refs to offer them as haves: {err:#}"
621            )))
622        })?;
623        let mut out = Vec::with_capacity(rows.len());
624        for row in rows {
625            if row.name == "HEAD" {
626                continue;
627            }
628            let Some(raw) = row.oid.as_deref() else {
629                continue;
630            };
631            let Ok(oid) = gix_hash::oid::try_from_bytes(raw) else {
632                return Err(gunnar_wire::Error::Store(gunnar_store::Error::backend(
633                    format!("{} holds an oid this store cannot decode", row.name),
634                )));
635            };
636            out.push((row.name, oid.to_owned()));
637        }
638        for (n, oid) in self.extra_haves.iter().enumerate() {
639            out.push((format!("refs/gunnar/resume/{n}"), *oid));
640        }
641        Ok(out)
642    }
643}
644
645/// **The queue's progress field, written from the wire and from the walk.**
646///
647/// Three publishers, one job record, and each of the three is the only place on
648/// the system that knows its own number:
649///
650/// | published by | what | when |
651/// |---|---|---|
652/// | [`FetchProgress::objects_total`] | the denominator | once, the instant the packfile header is read |
653/// | [`FetchProgress::pack_bytes`] | the byte count | as the stream is consumed, while the network is the bottleneck |
654/// | [`PackProgress::indexed`] | the numerator | once per object, as the engine's pack walk splits them |
655///
656/// # Why the numerator arrives from the other side of the seam
657///
658/// `gunnar-wire` reads bytes 8..12 of the header and can publish `(0, N)`
659/// before an object is stored — twelve bytes, and it is the reason the
660/// denominator is live at all. It cannot publish the numerator: counting
661/// objects as they pass means inflating every one of them, which is a second
662/// pack decoder, and gunnar has exactly one (one implementation, and `fetch_pack`'s own
663/// module doc refuses it in those words). The one decoder is the storage
664/// engine's walk, which has to split every entry anyway, so the count is free
665/// exactly there and nowhere else. This type is where the two halves meet.
666///
667/// # `total` is remembered here, and that is not a second counter
668///
669/// [`PackProgress::indexed`] carries its own `total` — the count the engine
670/// read from the same header — and it is *checked* against the one the wire
671/// announced rather than trusted, because two readings of one field that
672/// disagree mean the ingest is being handed a different pack from the one that
673/// was negotiated. `fetch_pack` makes the same comparison at the end, on the
674/// receipt; this is the same fact asserted while there is still a transfer to
675/// abandon.
676struct QueueProgress<'a> {
677    report: &'a Reporter<Mirror>,
678    total: AtomicU64,
679    /// A second sink, for a caller that has to see the *series* rather than the
680    /// job's latest reading. See [`sync_once_watching`].
681    watching: Option<&'a dyn git_storage_trait::PackProgress>,
682}
683
684impl QueueProgress<'_> {
685    /// **The one place a sync's progress pair is written.**
686    ///
687    /// All three publishers go through here — the header's `(0, N)`, the
688    /// engine's per-object readings, and [`finish`]'s `(N, N)` — so the job and
689    /// any watcher see the same values in the same order. Two routes to the
690    /// same field would be two places the series could differ, and the series
691    /// is the whole subject.
692    fn publish(&self, done: u64, total: u64) {
693        self.report.progress(done, total);
694        if let Some(watcher) = self.watching {
695            watcher.indexed(done, total);
696        }
697    }
698}
699
700impl FetchProgress for QueueProgress<'_> {
701    fn objects_total(&self, objects: u32) {
702        self.total.store(u64::from(objects), Ordering::SeqCst);
703        self.publish(0, u64::from(objects));
704    }
705
706    fn pack_bytes(&self, streamed: u64) {
707        self.report.report(|mirror| mirror.bytes = streamed);
708    }
709}
710
711impl git_storage_trait::PackProgress for QueueProgress<'_> {
712    fn indexed(&self, done: u64, total: u64) {
713        // The wire's denominator wins. An engine that read a different total
714        // from the same twelve bytes is a bug worth a log line, not a reason to
715        // renumber a progress bar mid-flight and leave a console watching a
716        // percentage that jumped sideways.
717        let announced = self.total.load(Ordering::SeqCst);
718        let total = if announced == 0 || announced == total {
719            total
720        } else {
721            tracing::warn!(
722                target: TARGET_GIT,
723                announced,
724                indexed_total = total,
725                "the packfile header the wire read and the one the store read declare \
726                 different object counts; the wire's is the one being rendered"
727            );
728            announced
729        };
730        self.publish(done, total);
731    }
732}
733
734// ── the fetch ────────────────────────────────────────────────────────────────
735
736/// **One sync, start to finish.** The [`crate::mirror_queue::MirrorRunner`]
737/// body.
738///
739/// Errors are `String` because they are going onto a job a console will read,
740/// not up a call stack — the same reason `gc_queue`'s runner returns one.
741pub fn sync_once(
742    stores: &Arc<ServerStores>,
743    order: &MirrorOrder,
744    credential: &Credential,
745    report: &Reporter<Mirror>,
746) -> Result<(), String> {
747    sync_once_watching(stores, order, credential, report, None)
748}
749
750/// [`sync_once`], with a **second** place the object count is published to.
751///
752/// # Why this door exists, and why it is not a test hook
753///
754/// [`Reporter::progress`] *overwrites*, by design — its own doc says a progress
755/// that accumulated would be a second counter to keep right — so the job record
756/// holds the latest reading and never the series. That is correct for a console,
757/// which only ever wants the latest reading, and it means the one thing that
758/// cannot be established from the job record afterwards is **whether the count
759/// moved at all**. A publisher that fires `(0, N)` and then `(N, N)` and a
760/// publisher that fires five hundred times in between leave the job in exactly
761/// the same state, so from the outside they are the same publisher.
762///
763/// That is measure-never-assert in its plainest form: a stub is
764/// indistinguishable from an
765/// implementation unless something can see it work — and it is precisely the
766/// defect this seam was opened to close: `(0, N) → (N, N)` was the shape for as
767/// long as no engine published from inside its pack walk. So the series gets a
768/// door, and the acceptance test stands at it. There is one other caller,
769/// [`sync_once`], which passes `None`.
770///
771/// The sink is *additional*: the job is written exactly as before, in the same
772/// order, whether or not anybody is watching. A test that changed what the
773/// production path publishes would be testing itself.
774pub fn sync_once_watching(
775    stores: &Arc<ServerStores>,
776    order: &MirrorOrder,
777    credential: &Credential,
778    report: &Reporter<Mirror>,
779    watching: Option<&dyn git_storage_trait::PackProgress>,
780) -> Result<(), String> {
781    let wire = classify(&order.upstream)?;
782    check_credential(wire, credential)?;
783    // Checked again on the worker, and not only at the door where
784    // `Mirror.Add` checked it: a registration made before this rule existed,
785    // or a name whose A record has moved since, reaches the dial through here
786    // and nowhere else.
787    check_destination(&order.upstream, stores.root())?;
788
789    // Resolved here, on the worker, and not at enqueue time: a repository can
790    // be deleted while its sync waits, and discovering that when the work
791    // starts is a FAILED job saying so, which is the truth.
792    let repo_ref = gunnar_core::resolve_repo_path(&order.path)
793        .map_err(|err| format!("`{}` is not a valid <namespace>/<name>: {err:#}", order.path))?;
794    let existing = stores
795        .catalog()
796        .get(&repo_ref)
797        .map_err(|err| format!("reading the catalog row for {}: {err:#}", order.path))?;
798    // **An import's repository is made HERE, by the first sync that reaches
799    // the upstream** (owner order 2026-10-18; `MirrorRecord::creates_repo`).
800    let to_create = existing.is_none()
801        && stores
802            .mirrors()
803            .get(&repo_ref)
804            .map_err(|err| format!("reading the registration for {}: {err:#}", order.path))?
805            .is_some_and(|record| record.creates_repo);
806    if existing.is_none() && !to_create {
807        return Err(format!("{} is no longer a live repository", order.path));
808    }
809    // **An encrypted repository is never fetched into** (wiring tracer B
810    // 2026-10-08): the ingest below runs no encrypted gate (`PushIngest::new`),
811    // so an upstream's plaintext would be stored and framed for the twin.
812    // `Mirror.Add` / `Repos.SetEncrypted` refuse making such a mirror; this is
813    // a registration older than that rule, or a flag set below the door.
814    if existing.as_ref().is_some_and(|row| row.encrypted) {
815        return Err(format!(
816            "{} is encrypted, so it is not fetched into: a mirror copies another server's \
817             plaintext, and an encrypted repository holds only what its members sealed \
818             (remove the mirror)",
819            order.path
820        ));
821    }
822    // **The namespace's room, asked of the same gates a push is held to**
823    // (2026-10-02). A suspended namespace or one already at its cap fetches
824    // nothing — refused here, before the dial, in the gate's own words — and
825    // the fetch is held to the tightest ceiling in the chain as it streams.
826    let ceiling = room_for(&stores.plan_gates(), repo_ref.namespace()).map_err(|(why, line)| {
827        report.report(|r| r.refused = why);
828        line
829    })?;
830
831    let auth = match credential {
832        Credential::Token { username, secret } if username.is_empty() => {
833            Some(format!("Bearer {secret}"))
834        }
835        Credential::Token { username, secret } => {
836            Some(format!("Basic {}", base64_basic(username, secret)))
837        }
838        _ => None,
839    };
840
841    let row = match existing {
842        Some(row) => row,
843        None => create_from_upstream(stores, &repo_ref, order, wire, credential, auth.clone())?,
844    };
845    let fetched = fetch_into(stores, &row, order, wire, credential, auth, ceiling, report, watching)
846        .and_then(|advertised| landed(stores, &row, order, &advertised).map(|()| advertised));
847    match &fetched {
848        Ok(advertised) => after_landing(stores, &repo_ref, &row, order, advertised),
849        Err(_) if to_create => discard_created(stores, &repo_ref, order),
850        Err(_) => {}
851    }
852    fetched.map(|_| ())
853}
854
855/// **A sync is DONE only when what the upstream advertised is here**
856/// (import-big, 2026-10-26).
857///
858/// Every branch and tag the upstream advertised must be a reference in this
859/// repository when the sync says it is done, and an upstream that advertised
860/// none is not an import that worked: it is an empty import, said as a
861/// failure. Read back from the store, after the apply, rather than inferred
862/// from the plan — the plan is what was meant, this is what landed. Before it,
863/// a fetch whose plan came out empty applied nothing and settled DONE over a
864/// repository with no references at all.
865fn landed(
866    stores: &Arc<ServerStores>,
867    row: &gunnar_core::model::Repo,
868    order: &MirrorOrder,
869    advertised: &[gunnar_wire::upload_pack::AdvertisedRef],
870) -> Result<(), String> {
871    let wanted: Vec<&str> = advertised
872        .iter()
873        .filter(|r| fetch_pack::mirror(&r.name) && r.target.is_some())
874        .map(|r| r.name.as_str())
875        .collect();
876    let upstream = scrub(&order.upstream);
877    if wanted.is_empty() {
878        return Err(format!(
879            "{upstream} has no branches or tags, so there was nothing to import into {}. \
880             Check the address, or push a first commit there and sync again.",
881            order.path
882        ));
883    }
884    let store = stores
885        .repo_store(row)
886        .map_err(|err| format!("opening {} to check what landed: {err:#}", order.path))?;
887    let held = GitOps::refs(store.serve())
888        .map_err(|err| format!("reading {}'s references to check what landed: {err:#}", order.path))?;
889    let missing: Vec<&str> = wanted
890        .iter()
891        .copied()
892        .filter(|name| !held.iter().any(|r| r.name == *name && r.oid.is_some()))
893        .collect();
894    if let Some(first) = missing.first() {
895        return Err(format!(
896            "the fetch from {upstream} ended without its references landing: {} of {} \
897             (first {first}) are not in {}. Nothing is reported as imported that is not \
898             here; sync again.",
899            missing.len(),
900            wanted.len(),
901            order.path
902        ));
903    }
904    Ok(())
905}
906
907/// **Ask the upstream what it is, then make the repository to match.**
908///
909/// One `ls-refs` at the upstream's own object format (`ls_remote_any`): the
910/// format becomes the repository's, and `HEAD`'s `symref-target` its default
911/// branch. Nothing is created when the upstream cannot be reached or refuses
912/// (a private repository without its token), so a failed first import leaves
913/// no repository behind.
914fn create_from_upstream(
915    stores: &Arc<ServerStores>,
916    repo_ref: &gunnar_core::RepoRef,
917    order: &MirrorOrder,
918    wire: Wire,
919    credential: &Credential,
920    auth: Option<String>,
921) -> Result<gunnar_core::model::Repo, String> {
922    let advertisement = ls_remote(stores, order, wire, credential, auth)?;
923    let hash = match advertisement.hash_kind {
924        gix_hash::Kind::Sha1 => gunnar_core::HashKind::Sha1,
925        _ => gunnar_core::HashKind::Sha256,
926    };
927    let mut new = gunnar_core::NewRepo::new(repo_ref.clone())
928        .hash_kind(hash)
929        .visibility(gunnar_core::Visibility::Private);
930    new.description = format!("mirror of {}", scrub(&order.upstream));
931    if let Some(branch) = advertised_head(&advertisement.refs) {
932        new = new.default_branch(branch);
933    }
934    tracing::info!(
935        target: TARGET_GIT,
936        event = "mirror.create_from_upstream",
937        repo = %order.path,
938        hash = ?hash,
939        "an import's repository is made at the upstream's object format and default branch"
940    );
941    let row = stores
942        .create_repo(new, false)
943        .map_err(|err| format!("creating {} for the import: {err:#}", order.path))?;
944    stores
945        .repo_store(&row)
946        .map_err(|err| format!("opening {} to mirror into it: {err:#}", order.path))?;
947    Ok(row)
948}
949
950/// **One `ls-refs` against the upstream**, at its own object format.
951fn ls_remote(
952    stores: &Arc<ServerStores>,
953    order: &MirrorOrder,
954    wire: Wire,
955    credential: &Credential,
956    auth: Option<String>,
957) -> Result<gunnar_wire::fetch_pack::Advertisement, String> {
958    let listed = |err: gunnar_wire::Error| {
959        format!(
960            "asking {} what it holds: {err}",
961            scrub(&order.upstream)
962        )
963    };
964    match wire {
965        Wire::Http => {
966            let (mut reader, mut writer) = open_http(&order.upstream, auth)?;
967            gunnar_wire::ls_remote_any(&mut reader, &mut writer).map_err(listed)
968        }
969        Wire::Ssh => dial_ssh(
970            &order.upstream,
971            credential,
972            &stores.root().join(MIRROR_KNOWN_HOSTS),
973            strict_host_keys(stores.root()),
974            |reader, writer| gunnar_wire::ls_remote_any(reader, writer),
975        )?
976        .map_err(listed),
977    }
978}
979
980/// **What an interrupted import already brought in**, offered back to the
981/// upstream as `have`s (import-big, 2026-10-26).
982///
983/// A repository that holds no branch and no tag is either new or an import
984/// that stopped between its pack landing and its references being written —
985/// a restart in the index, the longest step of a big import. The pack's bytes
986/// were durable before that step, and the indexer resumes them on its own after
987/// the restart; what was missing is only the reference write. So the resumed
988/// sync asks the upstream what it advertises, lets the index catch up, and
989/// offers every advertised tip this store already has as a `have`: the
990/// upstream then sends nothing it sent before, and a 6.6 GB kernel pack is not
991/// fetched (or stored) a second time. A new repository has nothing, and pays
992/// one `ls-refs` for finding out.
993#[allow(clippy::too_many_arguments)]
994fn already_here(
995    stores: &Arc<ServerStores>,
996    git: &dyn gunnar_store::git_storage_trait::GitServe,
997    git_dir: &std::path::Path,
998    order: &MirrorOrder,
999    wire: Wire,
1000    credential: &Credential,
1001    auth: Option<String>,
1002    report: &Reporter<Mirror>,
1003) -> Result<Vec<ObjectId>, String> {
1004    let held = GitOps::refs(git).map_err(|err| format!("reading {}'s references: {err:#}", order.path))?;
1005    if held.iter().any(|r| fetch_pack::mirror(&r.name) && r.oid.is_some()) {
1006        return Ok(Vec::new());
1007    }
1008    let advertisement = ls_remote(stores, order, wire, credential, auth)?;
1009    index_landed(git, git_dir, stores.engine(), report).map_err(|err| {
1010        format!("{} holds objects this server could not index: {err}", order.path)
1011    })?;
1012    let mut out = Vec::new();
1013    for advertised in &advertisement.refs {
1014        let Some(target) = advertised.target else { continue };
1015        if !fetch_pack::mirror(&advertised.name) || out.contains(&target) {
1016            continue;
1017        }
1018        if GitOps::has(git, target.as_bytes()).unwrap_or(false) {
1019            out.push(target);
1020        }
1021    }
1022    if !out.is_empty() {
1023        tracing::info!(
1024            target: TARGET_GIT,
1025            event = "mirror.sync.resumed",
1026            repo = %order.path,
1027            haves = out.len(),
1028            "an import that stopped after its pack landed resumes without fetching it again"
1029        );
1030    }
1031    Ok(out)
1032}
1033
1034/// `HEAD`'s branch, as the upstream advertised it, when it is a branch.
1035fn advertised_head(refs: &[gunnar_wire::upload_pack::AdvertisedRef]) -> Option<String> {
1036    refs.iter()
1037        .find(|r| r.name == "HEAD")
1038        .and_then(|r| r.symref_target.clone())
1039        .filter(|target| target.starts_with("refs/heads/") && gunnar_store::refname::validate(target).is_ok())
1040}
1041
1042/// A first import that failed after its repository was made: the repository
1043/// goes again, so nothing empty is left behind. Best effort — a removal that
1044/// fails is logged, and the registration still says the repository is to come.
1045fn discard_created(stores: &Arc<ServerStores>, repo_ref: &gunnar_core::RepoRef, order: &MirrorOrder) {
1046    let removed = stores
1047        .catalog()
1048        .delete(repo_ref)
1049        .map_err(|err| format!("{err:#}"))
1050        .and_then(|_| stores.purge_repo(repo_ref).map_err(|err| format!("{err:#}")));
1051    if let Err(err) = removed {
1052        tracing::warn!(
1053            target: TARGET_GIT,
1054            repo = %order.path,
1055            %err,
1056            "a failed first import could not remove the repository it had made"
1057        );
1058    }
1059}
1060
1061/// **What a landed sync owes the mirror besides its references**: the
1062/// upstream's default branch (when it holds that branch now) and whether the
1063/// upstream uses Git LFS. Neither fails the sync — the references landed.
1064fn after_landing(
1065    stores: &Arc<ServerStores>,
1066    repo_ref: &gunnar_core::RepoRef,
1067    row: &gunnar_core::model::Repo,
1068    order: &MirrorOrder,
1069    advertised: &[gunnar_wire::upload_pack::AdvertisedRef],
1070) {
1071    let mut row = row.clone();
1072    if let Some(branch) = advertised_head(advertised) {
1073        let held = stores
1074            .repo_store(&row)
1075            .ok()
1076            .and_then(|store| GitOps::refs(store.serve()).ok())
1077            .is_some_and(|refs| refs.iter().any(|r| r.name == branch && r.oid.is_some()));
1078        if held && branch != row.default_branch {
1079            match stores.catalog().set_default_branch(repo_ref, &branch) {
1080                Ok(moved) => {
1081                    row = moved;
1082                    if let Err(err) = stores.repoint_head(&row) {
1083                        tracing::warn!(target: TARGET_GIT, repo = %order.path, %err, "pointing HEAD at the upstream's default branch");
1084                    }
1085                }
1086                Err(err) => {
1087                    tracing::warn!(target: TARGET_GIT, repo = %order.path, %err, "taking the upstream's default branch");
1088                }
1089            }
1090        }
1091    }
1092    let lfs = uses_lfs(stores, &row);
1093    if let Err(err) = stores.mirrors().set_lfs_detected(repo_ref, lfs) {
1094        tracing::warn!(target: TARGET_GIT, repo = %order.path, %err, "recording whether the upstream uses LFS");
1095    }
1096}
1097
1098/// **Does the default branch route files through Git LFS?** A `filter=lfs`
1099/// in its root `.gitattributes` is the sign every LFS repository carries; the
1100/// files it names arrive as pointer files, and their content is not mirrored.
1101fn uses_lfs(stores: &Arc<ServerStores>, row: &gunnar_core::model::Repo) -> bool {
1102    use gunnar_store::git_storage_trait::ObjType;
1103    use gunnar_store::object;
1104    let Ok(store) = stores.repo_store(row) else {
1105        return false;
1106    };
1107    let git = store.serve();
1108    let hash = store.hash_kind();
1109    let Some(tip) = GitOps::refs(git)
1110        .ok()
1111        .and_then(|refs| refs.into_iter().find(|r| r.name == row.default_branch))
1112        .and_then(|r| r.oid)
1113        .and_then(|raw| ObjectId::try_from(&raw[..]).ok())
1114    else {
1115        return false;
1116    };
1117    let read = |id: &ObjectId, want: ObjType| match git.read(id.as_bytes()) {
1118        Ok(Some((kind, bytes))) if kind == want => Some(bytes),
1119        _ => None,
1120    };
1121    let Some(commit) = read(&tip, ObjType::Commit).and_then(|bytes| object::commit(&bytes, hash).ok()) else {
1122        return false;
1123    };
1124    let Some(tree) = read(&commit.tree, ObjType::Tree) else {
1125        return false;
1126    };
1127    let Ok(Some(entry)) = object::tree_entry_named(&tree, hash, b".gitattributes") else {
1128        return false;
1129    };
1130    read(&entry.oid, ObjType::Blob).is_some_and(|attributes| {
1131        String::from_utf8_lossy(&attributes)
1132            .lines()
1133            .any(|line| !line.trim_start().starts_with('#') && line.contains("filter=lfs"))
1134    })
1135}
1136
1137/// **The fetch and the apply**, into a repository that exists. Answers what
1138/// the upstream advertised, for [`after_landing`].
1139#[allow(clippy::too_many_arguments)]
1140fn fetch_into(
1141    stores: &Arc<ServerStores>,
1142    row: &gunnar_core::model::Repo,
1143    order: &MirrorOrder,
1144    wire: Wire,
1145    credential: &Credential,
1146    auth: Option<String>,
1147    ceiling: Option<gunnar_wire::PackCeiling>,
1148    report: &Reporter<Mirror>,
1149    watching: Option<&dyn git_storage_trait::PackProgress>,
1150) -> Result<Vec<gunnar_wire::upload_pack::AdvertisedRef>, String> {
1151    let row = row.clone();
1152    let store = stores
1153        .repo_store(&row)
1154        .map_err(|err| format!("opening {} to mirror into it: {err:#}", order.path))?;
1155    let git = store.serve();
1156    let hash_kind = store.hash_kind();
1157
1158    // Opened BEFORE the ingest, and that ordering is load-bearing for the same
1159    // reason it is on the push path: whether this repository has a protection
1160    // rule at all is what decides if the fetch pays for an object-set
1161    // enumeration.
1162    let protection = Protection::open(stores, &row, store.git_dir())?;
1163
1164    let extra_haves = already_here(stores, git, store.git_dir(), order, wire, credential, auth.clone(), report)?;
1165    let local = StoreRefs { git, extra_haves };
1166    let progress = QueueProgress {
1167        report,
1168        total: AtomicU64::new(0),
1169        watching,
1170    };
1171    // The ingest a PUSH lands through, unchanged. A fetched pack and a pushed
1172    // pack are the same bytes (`fetch_pack`'s module doc), so this is the same
1173    // storage authority and not a second one.
1174    //
1175    // The object set is asked for exactly when something reads it. A repository
1176    // with a protection rule has a gate that does (`require_signature` walks the
1177    // commits this fetch introduced), so the enumeration is `Required` and an
1178    // engine that cannot produce one must refuse rather than let the gate decide
1179    // on nothing. With no rule configured nobody reads it, and `NotNeeded` keeps
1180    // the fetch from waiting on index work no answer depends on.
1181    //
1182    // The last argument is the numerator's whole route: the ingest hands it to
1183    // the engine, the engine hands it to the pack walk, and the walk publishes
1184    // once per object into `progress` above — which is the same value the wire
1185    // is publishing the denominator and the byte count into. One sink, three
1186    // publishers, no second decoder anywhere on the path.
1187    let ingest = crate::transport::PushIngest::new(
1188        store.git_dir().to_path_buf(),
1189        hash_kind,
1190        stores.engine(),
1191        if protection.is_some() {
1192            crate::storage::ObjectSet::Required
1193        } else {
1194            crate::storage::ObjectSet::NotNeeded
1195        },
1196        Some(&progress),
1197        // A fetched pack is a transaction exactly as a pushed one is: framed before it lands.
1198        Some(crate::transport::PackFrame::of(stores, &row)),
1199    )
1200    // …and held to the data set's room the way a push is (cov-13).
1201    .with_space(stores.space_ledger());
1202
1203    let mut fetch =
1204        FetchPack::new(hash_kind, &fetch_pack::mirror, &local).with_progress(&progress);
1205    if let Some(ceiling) = ceiling {
1206        fetch = fetch.with_ceiling(ceiling);
1207    }
1208
1209    let outcome = match wire {
1210        Wire::Http => {
1211            let (mut reader, mut writer) = open_http(&order.upstream, auth)?;
1212            fetch.fetch(&ingest, &mut reader, &mut writer)
1213        }
1214        Wire::Ssh => dial_ssh(
1215            &order.upstream,
1216            credential,
1217            &stores.root().join(MIRROR_KNOWN_HOSTS),
1218            strict_host_keys(stores.root()),
1219            |reader, writer| fetch.fetch(&ingest, reader, writer),
1220        )?,
1221    }
1222    .map_err(|err| {
1223        // Cut at the namespace's ceiling: the plan's refusal, as a field.
1224        if matches!(err, gunnar_wire::Error::OverCeiling(_)) {
1225            report.report(|r| r.refused = crate::mirror_queue::Refusal::PackCap);
1226        }
1227        format!(
1228            "fetching {} from {}: {err}",
1229            order.path,
1230            scrub(&order.upstream)
1231        )
1232    })?;
1233
1234    // **Index the fetched pack ONCE, here, before a single read by oid**
1235    // (import-big, 2026-10-26) — the line `receive_pack::apply` has carried
1236    // since 2026-08-30 and this path never got.
1237    //
1238    // The znippy engine stores a pack's bytes before its oids exist and indexes
1239    // them behind the ack. `apply` reads by oid (the branch-protection gate, and
1240    // `put_refs_cas`, whose dangling-target check asks `has` for every new tip),
1241    // so without this its first read fell into the engine's read-time fallback:
1242    // a whole-pack resolve on this thread beside the indexer resolving the same
1243    // bytes. MEASURED on oden 2026-10-26, torvalds/linux from GitHub (6.60 GB,
1244    // 11 986 735 objects): the import landed only after the two resolvers had
1245    // driven the process to a 19.4 GB peak — past a 16 GB Mini, which is where
1246    // the owner's import of it died with nothing to say so.
1247    //
1248    // A refusal is the backend's own sentence (its memory watchdog gives up by
1249    // name, with what was available and what was needed): the pack is stored
1250    // and could not be indexed on this box, which is "this repository does not
1251    // fit here", never a sync that waits for ever or reads around it.
1252    if outcome.receipt.is_some() {
1253        index_landed(git, store.git_dir(), stores.engine(), report).map_err(|err| {
1254            format!(
1255                "{} was fetched and stored, and this server could not index it: {err}. Nothing \
1256                 was moved. The repository is larger than this server can take in; a server \
1257                 with more memory imports it.",
1258                order.path
1259            )
1260        })?;
1261    }
1262    apply(git, order, &outcome, &progress, protection.as_ref())?;
1263    Ok(outcome.advertised)
1264}
1265
1266/// **`GitServe::index_now`, with the index's own progress on the job.**
1267///
1268/// For a kernel-sized pack the index is minutes after the last byte arrived,
1269/// and a job that sat at `N/N` for all of them would read as stalled. So while
1270/// it runs, the engine's index row for this repository (`Engine::index_progress`,
1271/// the same row the dashboard reads) is published as the job's progress: the
1272/// objects indexed of the pack's total. When it returns the pair is the walk's
1273/// again (`finish`).
1274fn index_landed(
1275    git: &dyn gunnar_store::git_storage_trait::GitServe,
1276    git_dir: &std::path::Path,
1277    engine: crate::storage::Engine,
1278    report: &Reporter<Mirror>,
1279) -> Result<usize, String> {
1280    let done = std::sync::atomic::AtomicBool::new(false);
1281    std::thread::scope(|scope| {
1282        // A32-14, stays a scoped thread: it reads the index row for the length of one
1283        // `index_now` on the mirror job's own lane and is joined before this returns.
1284        scope.spawn(|| {
1285            while !done.load(Ordering::SeqCst) {
1286                if let Some((_, row)) = engine
1287                    .index_progress()
1288                    .into_iter()
1289                    .find(|(dir, row)| dir == git_dir && row.active && row.objects_total > 0)
1290                {
1291                    report.progress(row.objects_done, row.objects_total);
1292                }
1293                for _ in 0..10 {
1294                    if done.load(Ordering::SeqCst) {
1295                        return;
1296                    }
1297                    std::thread::sleep(std::time::Duration::from_millis(50));
1298                }
1299            }
1300        });
1301        let indexed = git.index_now().map_err(|err| format!("{err:#}"));
1302        done.store(true, Ordering::SeqCst);
1303        if let Ok(packs) = &indexed {
1304            tracing::info!(
1305                target: TARGET_GIT,
1306                event = "mirror.sync.indexed",
1307                packs,
1308                "the fetched pack was indexed once, before the references were checked"
1309            );
1310        }
1311        indexed
1312    })
1313}
1314
1315/// **How much a mirror sync into `namespace` may fetch, decided by the front
1316/// doors' own plan gates** ([`ServerStores::install_plan_gates`]).
1317///
1318/// The push path's questions, in its order (`gunnar_ssh::server`'s
1319/// `ask_plan`): the gate's verdict on a `Write`, then — for a write that grows
1320/// data, which a fetch is — `pack_quota` (is the namespace already at its
1321/// cap?) and the TIGHTEST `pack_ceiling` in the chain. The first `Refuse` is
1322/// the answer, verbatim. The ceiling is bound to the namespace's shared pack
1323/// ledger, marked here before any gate measured, so a mirror fetch and a push
1324/// landing at the same moment share the room instead of each spending it.
1325///
1326/// `Ok(None)` is "nothing caps it": a chain with no gate that answers, which is
1327/// the community appliance's ordinary state.
1328///
1329/// A refusal comes back with WHICH question refused it — the plan's verdict
1330/// on a write ([`crate::mirror_queue::Refusal::Plan`]) or the pack room
1331/// ([`crate::mirror_queue::Refusal::PackCap`]) — so the job can say so as a
1332/// field (`MirrorJob.refused_reason`).
1333pub(crate) fn room_for(
1334    gates: &[Arc<dyn gunnar_ssh::PlanGate>],
1335    namespace: &str,
1336) -> Result<Option<gunnar_wire::PackCeiling>, (crate::mirror_queue::Refusal, String)> {
1337    use gunnar_ssh::{PlanAction, PlanVerdict};
1338    if gates.is_empty() {
1339        return Ok(None);
1340    }
1341    // Taken BEFORE any gate measures, for the reason the push path gives:
1342    // only one of the two roundings is a quota that leaks.
1343    let mark = gunnar_lfs::AllowanceLedger::shared_pack().mark(namespace);
1344    let mut tightest: Option<gunnar_ssh::PackCeiling> = None;
1345    for gate in gates {
1346        if let PlanVerdict::Refuse(line) = gate.verdict(namespace, PlanAction::Write) {
1347            return Err((crate::mirror_queue::Refusal::Plan, line));
1348        }
1349        if let PlanVerdict::Refuse(line) = gate.pack_quota(namespace) {
1350            return Err((crate::mirror_queue::Refusal::PackCap, line));
1351        }
1352        if let Some(asked) = gate.pack_ceiling(namespace) {
1353            if tightest.as_ref().is_none_or(|held| asked.bytes < held.bytes) {
1354                tightest = Some(asked);
1355            }
1356        }
1357    }
1358    Ok(tightest.map(|ceiling| gunnar_wire::PackCeiling {
1359        bytes: ceiling.bytes,
1360        refusal: ceiling.refusal,
1361        ledger: Some(Arc::new(crate::transport::PackMark { mark })
1362            as Arc<dyn gunnar_wire::PackLedger>),
1363    }))
1364}
1365
1366/// **Apply the plan, in one transaction, against the values the fetch
1367/// reasoned about.**
1368///
1369/// See the module doc for why this is `put_refs_cas` and why a rejection is a
1370/// failure rather than a skip.
1371fn apply(
1372    git: &dyn gunnar_store::git_storage_trait::GitServe,
1373    order: &MirrorOrder,
1374    outcome: &FetchReport,
1375    progress: &QueueProgress<'_>,
1376    protection: Option<&Protection>,
1377) -> Result<(), String> {
1378    let zero = "0".repeat(outcome.hash_kind.len_in_hex());
1379
1380    // Owned first, because `RefCas` borrows its oids and the batch has to
1381    // outlive every one of them.
1382    // (name, old, new, peeled)
1383    type OwnedUpdate = (String, Option<Vec<u8>>, Option<Vec<u8>>, Option<Vec<u8>>);
1384    let mut owned: Vec<OwnedUpdate> = Vec::new();
1385    let mut updates: Vec<(String, String, String)> = Vec::new();
1386
1387    for plan in &outcome.updates {
1388        // **The name came off somebody else's wire, so it goes through the ref
1389        // name rules before it is written.**
1390        //
1391        // `plan.name` is the string an upstream's `ls-refs` answered with,
1392        // carried untouched from `fetch_pack`, and the only filter it passed on
1393        // the way is `fetch_pack::mirror`'s prefix test — which admits
1394        // `refs/heads/../../etc/passwd` exactly as readily as `refs/heads/main`.
1395        // Receive-pack puts every client-supplied name through
1396        // `gunnar_store::refname::validate` at the same point in its own path
1397        // (`receive_pack.rs`'s `funny refname`), and a fetch is the same
1398        // untrusted input arriving through a different door.
1399        //
1400        // It is checked HERE rather than relied on from the storage engine. The
1401        // memory backend does validate in `put_refs_cas`, so a name like this
1402        // would be refused there today — but that is a property of one engine,
1403        // and "a rule one backend enforces by accident is not a rule" is the
1404        // sentence `refname` itself is written under. The engines are swappable
1405        // (`GUNNAR_ENGINE`), and this is the one caller that can guarantee the
1406        // check ran.
1407        //
1408        // Only the names that came from the upstream. The prune half below
1409        // reads its names out of this repository's OWN refs, and a bad name
1410        // already in the store is one a prune should be able to delete rather
1411        // than one it must refuse to touch.
1412        gunnar_store::refname::validate(plan.name.as_bytes()).map_err(|err| {
1413            format!(
1414                "the upstream advertised a reference name gunnar will not write: {err}. Nothing \
1415                 was moved. A name is not a path, and this one would not have been a name."
1416            )
1417        })?;
1418        // The peel the remote already sent, rather than a peel computed here.
1419        // `ls-refs` was asked with `peel`, so an annotated tag arrived with the
1420        // commit it resolves to; recomputing it would be an object read per tag
1421        // for an answer already in hand (the tag re-peel, from the other side).
1422        let peeled = outcome
1423            .advertised
1424            .iter()
1425            .find(|advertised| advertised.name == plan.name)
1426            .and_then(|advertised| advertised.peeled)
1427            .map(|oid| oid.as_bytes().to_vec());
1428        updates.push((
1429            plan.name.clone(),
1430            plan.old.map(|old| old.to_string()).unwrap_or_else(|| zero.clone()),
1431            plan.new.to_string(),
1432        ));
1433        owned.push((
1434            plan.name.clone(),
1435            plan.old.map(|old| old.as_bytes().to_vec()),
1436            Some(plan.new.as_bytes().to_vec()),
1437            peeled,
1438        ));
1439    }
1440
1441    if order.prune {
1442        // **What the upstream no longer has.** Only over the same namespace the
1443        // ref plan covers (`refs/heads/*`, `refs/tags/*`): a prune that reached
1444        // wider would delete this box's own `refs/gunnar/*` bookkeeping on the
1445        // grounds that an upstream never advertised it.
1446        let held = GitOps::refs(git)
1447            .map_err(|err| format!("reading {}'s refs to prune them: {err:#}", order.path))?;
1448        for row in held {
1449            if !fetch_pack::mirror(&row.name) {
1450                continue;
1451            }
1452            if outcome
1453                .advertised
1454                .iter()
1455                .any(|advertised| advertised.name == row.name)
1456            {
1457                continue;
1458            }
1459            let Some(old) = row.oid else { continue };
1460            updates.push((row.name.clone(), hex(&old), zero.clone()));
1461            owned.push((row.name, Some(old), None, None));
1462        }
1463    }
1464
1465    if owned.is_empty() {
1466        finish(outcome, progress, Vec::new());
1467        return Ok(());
1468    }
1469
1470    // **The gate, between the plan and the write.** Before this line the batch
1471    // below was the one reference write on the box that no gate had seen; a
1472    // repository with no protection rule configured has no `Protection` at all,
1473    // so a mirror of an unprotected repository pays nothing for this.
1474    if let Some(protection) = protection {
1475        let gated: Vec<gunnar_policy::RefUpdate> = owned
1476            .iter()
1477            .map(|(name, old, new, _)| {
1478                // The null-oid spelling never appears here: `owned` already
1479                // carries absence as `None`, which is what `RefUpdate`'s three
1480                // constructors take. `try_from_bytes` cannot fail on a value the
1481                // store or the wire produced, and an oid this store cannot
1482                // decode is the one `StoreRefs::tips` already refuses by name.
1483                let decode = |raw: &Vec<u8>| {
1484                    gix_hash::oid::try_from_bytes(raw)
1485                        .map(gix_hash::oid::to_owned)
1486                        .map_err(|err| {
1487                            format!("{name} holds an oid this server cannot decode: {err}")
1488                        })
1489                };
1490                Ok(gunnar_policy::RefUpdate {
1491                    name: name.clone(),
1492                    old: old.as_ref().map(&decode).transpose()?,
1493                    new: new.as_ref().map(&decode).transpose()?,
1494                })
1495            })
1496            .collect::<Result<_, String>>()?;
1497        let new_objects = match outcome.receipt.as_ref() {
1498            // Nothing was fetched, so nothing was introduced. That is a measured
1499            // empty set and not an unasked question — see `FetchReport::receipt`.
1500            None => Ok(Vec::new()),
1501            Some(receipt) => receipt.new_objects.clone(),
1502        };
1503        protection.check(git, &gated, new_objects.as_deref().map_err(String::as_str))?;
1504    }
1505
1506    let batch: Vec<RefCas<'_>> = owned
1507        .iter()
1508        .map(|(name, old, new, peeled)| RefCas {
1509            name: name.clone(),
1510            old: old.as_deref(),
1511            new: new.as_deref(),
1512            peeled: peeled.clone(),
1513        })
1514        .collect();
1515
1516    // ONE call. Every edit or none — a loop with a break leaves what it already
1517    // applied applied, and a half-mirrored repository is a state no upstream
1518    // ever had.
1519    GitOps::put_refs_cas(git, &batch).map_err(|err| {
1520        format!(
1521            "the fetch succeeded and the reference update did not: {err:#}. Nothing was moved. \
1522             The usual cause is a push that landed on {} while the fetch was running; the \
1523             expected values came from the snapshot the fetch negotiated against, and \
1524             overwriting a concurrent push is the one thing a mirror must never do. Sync again.",
1525            order.path
1526        )
1527    })?;
1528
1529    tracing::info!(
1530        target: TARGET_GIT,
1531        event = "mirror.sync.applied",
1532        repo = %order.path,
1533        refs = updates.len(),
1534        objects = outcome.objects_done,
1535        bytes = outcome.pack_bytes,
1536        "a mirror sync moved this repository's references"
1537    );
1538
1539    finish(outcome, progress, updates);
1540    Ok(())
1541}
1542
1543/// **Close the job's numbers out**, on both paths out of [`apply`].
1544///
1545/// One function and not two copies, because the pair below is the whole of what
1546/// a console reads when a sync ends and the two paths differ only in whether
1547/// any reference moved.
1548///
1549/// The `progress` call is the second half of the pair `FetchProgress` opened.
1550/// `objects_total` published `(0, N)` the instant the packfile header arrived;
1551/// **this publishes `(N, N)`**, and without it a finished job sits for ever at
1552/// nought per cent of a total it knows — which is worse than no progress bar,
1553/// because it reads as a transfer that stalled. `fetch_pack` has already
1554/// refused the transfer if the declared count and the indexed count disagree,
1555/// so `objects_done` is the same `N` and this cannot report a completion the
1556/// store did not make.
1557///
1558/// A fetch that wanted nothing — the ordinary answer for a poll of an idle
1559/// upstream — has no header, no `N`, and therefore **no progress at all**: the
1560/// pair stays `None`, and a caller reads `has_progress = false` rather than a
1561/// fabricated `0/0`.
1562fn finish(
1563    outcome: &FetchReport,
1564    progress: &QueueProgress<'_>,
1565    updates: Vec<(String, String, String)>,
1566) {
1567    if outcome.receipt.is_some() {
1568        let total = u64::from(outcome.objects_total);
1569        progress.publish(u64::from(outcome.objects_done), total);
1570    }
1571    progress.report.report(|mirror| {
1572        mirror.objects = u64::from(outcome.objects_done);
1573        mirror.bytes = outcome.pack_bytes;
1574        mirror.updates = updates;
1575    });
1576}
1577
1578/// Lower-case hex, the spelling `gunnar.v1.RefUpdate` uses.
1579fn hex(raw: &[u8]) -> String {
1580    let mut out = String::with_capacity(raw.len() * 2);
1581    for byte in raw {
1582        use std::fmt::Write as _;
1583        let _ = write!(out, "{byte:02x}");
1584    }
1585    out
1586}
1587
1588/// `base64(user:secret)`, the Basic-auth pair.
1589///
1590/// Written out rather than pulled in: the whole alphabet is 64 characters and
1591/// the encoder is nine lines, against a dependency that would be in the
1592/// appliance's graph for this one call. It is not a general base64 — it does
1593/// exactly the one thing, and `mirror_http`'s tests assert on the bytes it
1594/// produces.
1595fn base64_basic(user: &str, secret: &str) -> String {
1596    const ALPHABET: &[u8; 64] =
1597        b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
1598    let raw = format!("{user}:{secret}");
1599    let raw = raw.as_bytes();
1600    let mut out = String::with_capacity(raw.len().div_ceil(3) * 4);
1601    for chunk in raw.chunks(3) {
1602        let b = [
1603            chunk[0],
1604            chunk.get(1).copied().unwrap_or(0),
1605            chunk.get(2).copied().unwrap_or(0),
1606        ];
1607        let packed = (u32::from(b[0]) << 16) | (u32::from(b[1]) << 8) | u32::from(b[2]);
1608        out.push(ALPHABET[(packed >> 18) as usize & 63] as char);
1609        out.push(ALPHABET[(packed >> 12) as usize & 63] as char);
1610        out.push(if chunk.len() > 1 {
1611            ALPHABET[(packed >> 6) as usize & 63] as char
1612        } else {
1613            '='
1614        });
1615        out.push(if chunk.len() > 2 {
1616            ALPHABET[packed as usize & 63] as char
1617        } else {
1618            '='
1619        });
1620    }
1621    out
1622}
1623
1624// ── the ssh half ─────────────────────────────────────────────────────────────
1625
1626/// **Dial `git-upload-pack` over SSH and hand the channel's two halves to
1627/// `run`.**
1628///
1629/// `gunnar_client::connect_upload_pack` is the fetch-side twin of the
1630/// `connect_receive_pack` replication already pushes with, over
1631/// `nornir-git`'s russh transport. There is no second dialler here and there
1632/// must not be: that transport owns the connect, the publickey exchange, the
1633/// `known_hosts` policy, the blocking bridge and — load-bearing — the `Drop`
1634/// that tears the channel down *inside* the driving runtime.
1635///
1636/// # The key is a file for the length of one dial, and then it is not
1637///
1638/// `nornir_git::ssh::SshAuth` authenticates from a **path**. gunnar's mirror
1639/// credential is sealed bytes in the control database, because a private key
1640/// sitting in the clear on `/data` is the thing that store exists to prevent.
1641/// The two are bridged by a [`tempfile::NamedTempFile`] created `0600` and
1642/// dropped — and therefore unlinked — before this function returns, on every
1643/// path out of it including a panic.
1644///
1645/// The alternative was an in-memory key API on `nornir-git`, which is the right
1646/// long-term shape, and it is a change to a crate outside this repository. It
1647/// is named here rather than left as a silent compromise.
1648#[cfg(feature = "replication")]
1649fn dial_ssh<T>(
1650    url: &str,
1651    credential: &Credential,
1652    known_hosts: &std::path::Path,
1653    strict: bool,
1654    run: impl FnOnce(&mut dyn BufRead, &mut dyn Write) -> T,
1655) -> Result<T, String> {
1656    use gunnar_client::ssh::{HostKeyPolicy, SshOptions};
1657    use std::io::BufReader;
1658
1659    let mut options = SshOptions::new();
1660    // A server has no ssh-agent and no interactive user. Offering the agent
1661    // would be offering identities nobody configured for this mirror — the
1662    // silent substitution `gunnar_client::resolve_identity` exists to forbid.
1663    options.use_agent = false;
1664
1665    // Held for the whole of `run`: dropping it unlinks the file, and the dial
1666    // reads it during `connect_upload_pack`.
1667    let key_file = match credential {
1668        Credential::SshKey(bytes) => {
1669            let mut file = tempfile::Builder::new()
1670                .prefix("gunnar-mirror-key-")
1671                .tempfile()
1672                .map_err(|err| format!("no room to materialise the mirror's ssh key: {err}"))?;
1673            #[cfg(unix)]
1674            {
1675                use std::os::unix::fs::PermissionsExt as _;
1676                std::fs::set_permissions(file.path(), std::fs::Permissions::from_mode(0o600))
1677                    .map_err(|err| {
1678                        format!("the mirror's ssh key could not be made private: {err}")
1679                    })?;
1680            }
1681            file.write_all(bytes)
1682                .and_then(|()| file.flush())
1683                .map_err(|err| format!("writing the mirror's ssh key: {err}"))?;
1684            options.key_path = Some(file.path().to_path_buf());
1685            Some(file)
1686        }
1687        // No key named means the ladder, which on a server finds nothing; that
1688        // is the honest behaviour for a public upstream over ssh and it is what
1689        // `gunnar-client` warns about at start-up.
1690        _ => None,
1691    };
1692    // Trust-on-first-use against a `known_hosts` under the data root is what
1693    // replication does when an operator has not said otherwise, and a
1694    // mirror is dialling the same kind of box. See [`MIRROR_KNOWN_HOSTS`] for
1695    // why the file is NOT `~/.ssh/known_hosts`.
1696    //
1697    // **Unless the box says otherwise**, which it could not before: the policy
1698    // was this literal and nothing else, so a first sync trusted whatever
1699    // answered on the upstream's address, on a path that is about to offer it a
1700    // private key. See [`MIRROR_STRICT_HOST_KEY`].
1701    options.host_key = if strict {
1702        HostKeyPolicy::StrictKnownHosts
1703    } else {
1704        HostKeyPolicy::TofuKnownHosts
1705    };
1706    options.known_hosts = Some(known_hosts.to_path_buf());
1707
1708    let location = nornir_git::ssh::parse_ssh_url(url)
1709        .map_err(|err| format!("{} is not an ssh url: {err}", scrub(url)))?;
1710    let mut channel = gunnar_client::connect_upload_pack(&location, &options).map_err(|err| {
1711        // `{:#}` over an `anyhow::Error`, and not `{err}`: `gunnar_client::Error`
1712        // is a thiserror enum whose `Ssh` variant Displays as the bare word
1713        // "ssh" and carries the real sentence — which host key, which policy —
1714        // as its `source`. A FAILED job reading `opening git-upload-pack on …:
1715        // ssh` tells an operator nothing they can act on, and the reason was
1716        // one formatting flag away the whole time.
1717        format!(
1718            "opening git-upload-pack on {}: {:#}",
1719            scrub(url),
1720            anyhow::Error::new(err)
1721        )
1722    })?;
1723    let out = {
1724        let (reader, writer) = channel.io();
1725        let mut reader = BufReader::new(reader);
1726        run(&mut reader, writer)
1727    };
1728    drop(key_file);
1729    Ok(out)
1730}
1731
1732/// **What a build with no ssh dialler says**, rather than what it does.
1733///
1734/// `replication` is the feature that carries `gunnar-client`'s wire half and
1735/// `nornir-git`'s russh transport; without it there is no dialler in the
1736/// binary. A mirror registered against an `ssh://` remote on such a build is a
1737/// registration that can never run, and it says so at the one place it can be
1738/// acted on — a FAILED job naming the flag — rather than failing as a
1739/// connection error that sends an operator to look at the network.
1740///
1741/// The smart-HTTP half is behind its own feature, `mirror-http` (A32-18), which every
1742/// appliance row names: [`NO_MIRROR_HTTP`] is its refusal.
1743#[cfg(not(feature = "replication"))]
1744fn dial_ssh<T>(
1745    url: &str,
1746    _credential: &Credential,
1747    _known_hosts: &std::path::Path,
1748    _strict: bool,
1749    _run: impl FnOnce(&mut dyn BufRead, &mut dyn Write) -> T,
1750) -> Result<T, String> {
1751    Err(format!(
1752        "{} is an ssh remote and this build carries no ssh dialler: it was compiled without \
1753         `--features replication`, which is what links `gunnar-client`'s russh transport. \
1754         Mirror from the upstream's `https://` URL, or use a build that has the feature.",
1755        scrub(url)
1756    ))
1757}
1758
1759/// **Take any userinfo out of a URL before it is shown to anybody.**
1760///
1761/// `https://user:token@host/path` is a legal way to write a remote and an
1762/// operator may well have written one. Everything between `//` and the last
1763/// `@` of the authority is replaced, so a token that arrived in the URL cannot
1764/// leave through an error string.
1765///
1766/// Not a parser: this runs in an error path, and a URL this cannot make sense
1767/// of is returned with the whole authority elided rather than passed through.
1768pub(crate) fn scrub(url: &str) -> String {
1769    let Some((scheme, rest)) = url.split_once("://") else {
1770        return url.to_owned();
1771    };
1772    let (authority, path) = match rest.find('/') {
1773        Some(cut) => (&rest[..cut], &rest[cut..]),
1774        None => (rest, ""),
1775    };
1776    match authority.rfind('@') {
1777        Some(cut) => format!("{scheme}://<redacted>@{}{path}", &authority[cut + 1..]),
1778        None => url.to_owned(),
1779    }
1780}
1781
1782/// **An `https://` remote's two halves** ([`crate::mirror_http`], feature `mirror-http`).
1783#[cfg(feature = "mirror-http")]
1784fn open_http(url: &str, auth: Option<String>) -> Result<(crate::mirror_http::HttpRead, crate::mirror_http::HttpWrite), String> {
1785    crate::mirror_http::open(url, auth)
1786}
1787
1788/// The refusal an `https://` remote gets in a build without `mirror-http` (A32-18): the
1789/// airgap default has no HTTP client, and says so where it can be acted on.
1790pub const NO_MIRROR_HTTP: &str = "this build carries no HTTP client: it was compiled without gunnar-server's `mirror-http` feature (the appliance rows name it)";
1791
1792#[cfg(not(feature = "mirror-http"))]
1793fn open_http(url: &str, _auth: Option<String>) -> Result<(std::io::Empty, std::io::Sink), String> {
1794    Err(format!("{} is an https remote and {NO_MIRROR_HTTP}. Mirror from the upstream's ssh:// URL, or use a build that has the feature.", scrub(url)))
1795}
1796
1797#[cfg(test)]
1798mod tests {
1799    use super::*;
1800
1801    /// **The userinfo scrub is the last thing between a token in a URL and the
1802    /// log.** Asserted on the bytes, because "it does not appear" is the whole
1803    /// property.
1804    /// ★ A32-18: the airgap default carries no HTTP client, so an `https://` remote is refused
1805    /// by name (and its token never shown), never dialled.
1806    #[cfg(not(feature = "mirror-http"))]
1807    #[test]
1808    fn an_https_remote_is_refused_by_name_in_a_build_without_mirror_http() {
1809        let e = open_http("https://user:tok3n@forge.mock.invalid/ns/repo.git", Some("Bearer tok3n".into())).err().expect("refused");
1810        assert!(e.contains(NO_MIRROR_HTTP) && e.contains("mirror-http"), "{e}");
1811        assert!(!e.contains("tok3n"), "{e}");
1812    }
1813
1814    #[test]
1815    fn a_url_with_userinfo_never_shows_it() {
1816        let scrubbed = scrub("https://alice:ghp_SUPERSECRET@example.test/ns/repo.git");
1817        assert!(
1818            !scrubbed.contains("ghp_SUPERSECRET"),
1819            "the token survived the scrub: {scrubbed}"
1820        );
1821        assert!(
1822            !scrubbed.contains("alice"),
1823            "the whole userinfo goes, not just the password half: {scrubbed}"
1824        );
1825        assert_eq!(scrubbed, "https://<redacted>@example.test/ns/repo.git");
1826    }
1827
1828    /// A URL with no userinfo is left exactly as the operator wrote it — an
1829    /// error that mangled the ordinary case would be worse than no scrub.
1830    #[test]
1831    fn a_plain_url_is_untouched() {
1832        let url = "https://example.test/ns/repo.git";
1833        assert_eq!(scrub(url), url);
1834        assert_eq!(scrub("ssh://git@example.test/ns/repo.git"), "ssh://<redacted>@example.test/ns/repo.git");
1835    }
1836
1837
1838    /// **Both transports are recognised from the URL alone**, and the scp-like
1839    /// form is ssh — the shape every forge prints on its clone button.
1840    #[test]
1841    fn a_remote_url_names_its_transport() {
1842        assert_eq!(classify("ssh://git@host/ns/repo.git"), Ok(Wire::Ssh));
1843        assert_eq!(classify("git+ssh://git@host/ns/repo.git"), Ok(Wire::Ssh));
1844        assert_eq!(classify("git@host:ns/repo.git"), Ok(Wire::Ssh));
1845        assert_eq!(classify("https://host/ns/repo.git"), Ok(Wire::Http));
1846        assert_eq!(classify("http://127.0.0.1:9418/ns/repo.git"), Ok(Wire::Http));
1847    }
1848
1849    /// A remote gunnar cannot dial is refused when it is REGISTERED, not on the
1850    /// first sync — and the refusal names the two forms that work.
1851    #[test]
1852    fn a_remote_that_is_neither_is_refused_by_name() {
1853        let err = classify("/srv/git/ns/repo.git").expect_err("a local path is not a remote");
1854        assert!(err.contains("ssh://") && err.contains("https://"), "{err}");
1855        let err = classify("git://host/ns/repo.git").expect_err("the dumb git protocol");
1856        assert!(err.contains("scheme gunnar cannot mirror from"), "{err}");
1857    }
1858
1859    /// **The mismatch is refused by name, in both directions.**
1860    ///
1861    /// The failure being prevented is not an error — it is a 401 (or an SSH
1862    /// authentication failure) against an operator who can see that a
1863    /// credential IS stored, and who will therefore rotate a key that was never
1864    /// the problem.
1865    #[test]
1866    fn a_credential_of_the_wrong_kind_is_refused_by_name() {
1867        let key = Credential::SshKey(b"-----BEGIN OPENSSH PRIVATE KEY-----".to_vec());
1868        let token = Credential::Token {
1869            username: String::new(),
1870            secret: "ghp_SECRET".to_owned(),
1871        };
1872
1873        let err = check_credential(Wire::Http, &key).expect_err("a key cannot ride on http");
1874        assert!(err.contains("smart http") && err.contains("ssh private key"), "{err}");
1875        assert!(!err.contains("BEGIN OPENSSH"), "the refusal quoted the key: {err}");
1876
1877        let err = check_credential(Wire::Ssh, &token).expect_err("a token cannot ride on ssh");
1878        assert!(err.contains("ssh") && err.contains("a token"), "{err}");
1879        assert!(!err.contains("ghp_SECRET"), "the refusal quoted the token: {err}");
1880
1881        // …and the matching pairs, plus the public case, are all fine.
1882        check_credential(Wire::Ssh, &key).expect("a key over ssh");
1883        check_credential(Wire::Http, &token).expect("a token over http");
1884        check_credential(Wire::Ssh, &Credential::None).expect("a public ssh upstream");
1885        check_credential(Wire::Http, &Credential::None).expect("a public http upstream");
1886    }
1887
1888    /// **The authority, out of each of the three shapes** — and the default
1889    /// port with it, because the port is half of what gets dialled.
1890    #[test]
1891    fn a_remote_url_names_the_machine_it_would_dial() {
1892        for (url, want) in [
1893            ("https://host/ns/repo.git", ("host".to_owned(), 443)),
1894            ("http://host/ns/repo.git", ("host".to_owned(), 80)),
1895            ("http://host:9418/ns/repo.git", ("host".to_owned(), 9418)),
1896            ("ssh://git@host/ns/repo.git", ("host".to_owned(), 22)),
1897            ("ssh://git@host:2222/ns/repo.git", ("host".to_owned(), 2222)),
1898            ("git@host:ns/repo.git", ("host".to_owned(), 22)),
1899            // The userinfo is not the host, and a `@` inside it must not become
1900            // one either.
1901            ("https://user:pass@host/ns/repo.git", ("host".to_owned(), 443)),
1902            // An IPv6 literal is all colons; the brackets are what separates the
1903            // address from the port, and splitting on the first colon would
1904            // yield `[`.
1905            ("ssh://git@[::1]:2222/ns/repo.git", ("::1".to_owned(), 2222)),
1906            (
1907                "https://[2606:4700::1111]/ns/repo.git",
1908                ("2606:4700::1111".to_owned(), 443),
1909            ),
1910            ("git@[::1]:ns/repo.git", ("::1".to_owned(), 22)),
1911        ] {
1912            assert_eq!(authority(url), Ok(want.clone()), "{url}");
1913        }
1914        assert!(authority("https:///ns/repo.git").is_err(), "no host at all");
1915    }
1916
1917    /// **Every range a caller must not be able to point this server at**, and
1918    /// the two that must still work.
1919    ///
1920    /// Asserted on [`address_verdict`] rather than on [`check_destination`]
1921    /// because the addresses are handed in: no resolver runs, so this is the
1922    /// same answer on a laptop, on a build box with a corporate DNS and in a
1923    /// container with none.
1924    #[test]
1925    fn a_mirror_refuses_every_address_only_the_server_can_reach() {
1926        for addr in [
1927            "127.0.0.1",
1928            "0.0.0.0",
1929            "10.0.0.5",
1930            "172.16.0.1",
1931            "192.168.1.1",
1932            // The one the finding names: cloud instance metadata.
1933            "169.254.169.254",
1934            "100.64.0.1",
1935            "192.0.0.1",
1936            "198.18.0.1",
1937            "240.0.0.1",
1938            "255.255.255.255",
1939            "224.0.0.1",
1940            "::1",
1941            "::",
1942            "fc00::1",
1943            "fe80::1",
1944            "ff02::1",
1945            // …written the long way. Without the mapped-address arm this one
1946            // walks past every v6 test and is allowed.
1947            "::ffff:127.0.0.1",
1948            "::ffff:169.254.169.254",
1949        ] {
1950            let ip: std::net::IpAddr = addr.parse().expect("a literal");
1951            let err = address_verdict("upstream.invalid", &[ip])
1952                .expect_err("{addr} must not be dialled on a caller's say-so");
1953            assert!(
1954                err.contains(addr) || err.contains(&ip.to_string()),
1955                "{addr}: {err}"
1956            );
1957            assert!(err.contains(MIRROR_ALLOW_PRIVATE), "{addr}: {err}");
1958        }
1959
1960        for addr in ["93.184.216.34", "8.8.8.8", "2606:4700:4700::1111"] {
1961            let ip: std::net::IpAddr = addr.parse().expect("a literal");
1962            address_verdict("upstream.invalid", &[ip])
1963                .unwrap_or_else(|err| panic!("{addr} is an ordinary public address: {err}"));
1964        }
1965    }
1966
1967    /// **One bad answer out of several is a refusal.** A name that resolves to a
1968    /// routable address and a loopback address is the rebinding shape, and a
1969    /// check that took the first answer, or the best one, would pass it.
1970    #[test]
1971    fn one_private_answer_among_several_refuses_the_whole_name() {
1972        let addrs: Vec<std::net::IpAddr> = ["93.184.216.34", "127.0.0.1"]
1973            .iter()
1974            .map(|a| a.parse().expect("a literal"))
1975            .collect();
1976        let err = address_verdict("both.invalid", &addrs).expect_err("one of them is loopback");
1977        assert!(err.contains("127.0.0.1"), "{err}");
1978
1979        // And nothing at all is not "fine": a name with no address is refused
1980        // rather than allowed through to a dial that would fail anyway.
1981        assert!(address_verdict("nowhere.invalid", &[]).is_err());
1982    }
1983
1984    /// The Basic pair is encoded the way every HTTP client encodes it. Checked
1985    /// against a value anybody can verify by hand rather than against this
1986    /// function's own output.
1987    #[test]
1988    fn basic_auth_is_ordinary_base64() {
1989        assert_eq!(base64_basic("Aladdin", "open sesame"), "QWxhZGRpbjpvcGVuIHNlc2FtZQ==");
1990        assert_eq!(base64_basic("a", "b"), "YTpi");
1991        assert_eq!(base64_basic("ab", "b"), "YWI6Yg==");
1992    }
1993}