git_attest/
notes.rs

1//! The notes refs as origin's mirror (SPEC.md, "Which refs are read").
2
3use std::path::Path;
4use std::process::{Command, Stdio};
5
6use crate::format::{INPUTS_REF, NOTES_REF};
7use crate::git;
8
9/// What became of one notes ref. SPEC.md, "Lookup".
10///
11/// `refs/notes/amont-attest[-inputs]` are ORIGIN'S MIRROR, and origin is the
12/// only place an attestation can be revoked, so only a mirror that is
13/// origin's ref right now — or a repository with no origin at all, which is
14/// for fixtures and local use — is judged.
15#[derive(Clone, Copy, Debug, PartialEq)]
16pub enum Mirror {
17    /// Fetched: the mirror is origin's ref.
18    Fresh,
19    /// No remote named origin: the local ref is judged as it is.
20    NoOrigin,
21    /// Origin answered and has no such ref: the mirror was deleted.
22    Deleted,
23    /// As `Deleted`, but the delete failed: the stale mirror is not judged.
24    Undeletable,
25    /// Origin is configured and did not answer: not judged.
26    Unreachable,
27    /// As `Unreachable`, because the deadline cut the call off.
28    TimedOut,
29    /// Origin's copy was fetched but could not be made the local ref (a
30    /// stale lock, say): the stale local copy is not judged.
31    Unwritable,
32}
33
34impl Mirror {
35    pub fn as_str(self) -> &'static str {
36        match self {
37            Mirror::Fresh => "fetched",
38            Mirror::NoOrigin => "no-origin",
39            Mirror::Deleted => "absent",
40            Mirror::Undeletable => "undeletable",
41            Mirror::Unreachable | Mirror::TimedOut => "unreachable",
42            Mirror::Unwritable => "unwritable",
43        }
44    }
45    pub fn judged(self) -> bool {
46        matches!(self, Mirror::Fresh | Mirror::NoOrigin)
47    }
48}
49
50/// Delete the mirror of `notes_ref` because origin has none.
51fn drop_mirror(root: &Path, loud: &mut Vec<String>, notes_ref: &str) -> Mirror {
52    let full = format!("refs/notes/{notes_ref}");
53    let Some(oid) = git::stdout_in(root, &["rev-parse", "--verify", "--quiet", &full]) else {
54        return Mirror::Deleted;
55    };
56    if git::succeeds_in(root, &["update-ref", "-d", &full, &oid]) {
57        loud.push(format!(
58            "origin has no {full}; deleted the local mirror (was {oid}); restore: git update-ref {full} {oid}"
59        ));
60        Mirror::Deleted
61    } else {
62        Mirror::Undeletable
63    }
64}
65
66/// Where fetches land before they become the mirror: outside `refs/notes/`,
67/// one directory per process id, so no notes push or pruning fetch ever
68/// touches them.
69pub(crate) const TMP_NS: &str = "refs/attest-tmp/";
70
71/// Remove the throwaways of runs killed before their cleanup — only those
72/// whose process is gone, so a concurrent run in the same clone keeps its
73/// own. Where liveness cannot be asked, nothing is swept: a leftover outside
74/// `refs/notes/` is inert.
75fn sweep_tmp_refs(root: &Path) {
76    let Some(refs) = git::stdout_in(root, &["for-each-ref", "--format=%(refname)", TMP_NS]) else {
77        return;
78    };
79    let me = std::process::id().to_string();
80    for r in refs.lines() {
81        let Some(pid) = r
82            .strip_prefix(TMP_NS)
83            .and_then(|rest| rest.split('/').next())
84            .filter(|p| !p.is_empty() && p.bytes().all(|b| b.is_ascii_digit()))
85        else {
86            continue;
87        };
88        if pid != me && process_gone(pid) {
89            let _ = git::succeeds_in(root, &["update-ref", "-d", r]);
90        }
91    }
92}
93
94/// True only on a positive "no such process" (read in the C locale); any
95/// other answer, or none, counts as alive, so nothing live is swept.
96fn process_gone(pid: &str) -> bool {
97    if !cfg!(unix) {
98        return false;
99    }
100    Command::new("kill")
101        .args(["-0", pid])
102        .env("LC_ALL", "C")
103        .stdin(Stdio::null())
104        .stdout(Stdio::null())
105        .stderr(Stdio::piped())
106        .output()
107        .is_ok_and(|o| {
108            !o.status.success()
109                && String::from_utf8_lossy(&o.stderr)
110                    .to_lowercase()
111                    .contains("no such process")
112        })
113}
114
115/// Make the fetched copy in `tmp` the mirror of `notes_ref`, by a
116/// compare-and-swap against `old` (what was there before the fetch). A kill
117/// can only ever half-write the throwaway, never the mirror. When the swap
118/// fails, the mirror is judged only if it now holds EXACTLY origin's copy (a
119/// concurrent run got there first, or origin is unchanged under a lock).
120fn adopt(root: &Path, notes_ref: &str, tmp: &str, old: Option<&str>) -> Mirror {
121    let full = format!("refs/notes/{notes_ref}");
122    let Some(new) = git::stdout_in(root, &["rev-parse", "--verify", "--quiet", tmp]) else {
123        return Mirror::Unwritable;
124    };
125    if git::succeeds_in(root, &["update-ref", &full, &new, old.unwrap_or("")]) {
126        return Mirror::Fresh;
127    }
128    match git::stdout_in(root, &["rev-parse", "--verify", "--quiet", &full]) {
129        Some(now) if now == new => Mirror::Fresh,
130        _ => Mirror::Unwritable,
131    }
132}
133
134/// Bring both mirrors in line with origin: one fetch of both refs into
135/// throwaways and, when that fails without timing out, one `ls-remote` to
136/// tell "origin has no such ref" from "origin did not answer" — then, if some
137/// refs exist, one more fetch of those. A silent origin costs one call (15 s
138/// at most), a dead one two (30 s), the worst case three (45 s).
139pub fn sync_mirrors(root: &Path, loud: &mut Vec<String>) -> (Mirror, Mirror) {
140    if !git::succeeds_in(root, &["remote", "get-url", "origin"]) {
141        return (Mirror::NoOrigin, Mirror::NoOrigin);
142    }
143    sweep_tmp_refs(root);
144    let pid = std::process::id();
145    let full = |r: &str| format!("refs/notes/{r}");
146    let tmp = |r: &str| format!("{TMP_NS}{pid}/{r}");
147    let spec = |r: &str| format!("+{}:{}", full(r), tmp(r));
148    // Our own throwaways, and any lock a dead run with our pid left on them.
149    // Known limits: a remove that fails leaves the fetch to fail and read as
150    // unreachable (safe: nothing judged); a killed fetch's lock with no ref
151    // beside it is not swept, only replaced when its pid comes round again.
152    for r in [NOTES_REF, INPUTS_REF] {
153        let _ = git::succeeds_in(root, &["update-ref", "-d", &tmp(r)]);
154        if let Some(lock) = git::stdout_in(
155            root,
156            &["rev-parse", "--git-path", &format!("{}.lock", tmp(r))],
157        ) {
158            // `--git-path` answers relative to where git ran.
159            let _ = std::fs::remove_file(root.join(lock));
160        }
161    }
162    let old_main = git::stdout_in(
163        root,
164        &["rev-parse", "--verify", "--quiet", &full(NOTES_REF)],
165    );
166    let old_inputs = git::stdout_in(
167        root,
168        &["rev-parse", "--verify", "--quiet", &full(INPUTS_REF)],
169    );
170    let adopt_both = || {
171        (
172            adopt(root, NOTES_REF, &tmp(NOTES_REF), old_main.as_deref()),
173            adopt(root, INPUTS_REF, &tmp(INPUTS_REF), old_inputs.as_deref()),
174        )
175    };
176    let states = match git::remote(
177        root,
178        &[
179            "fetch",
180            "--quiet",
181            "origin",
182            &spec(NOTES_REF),
183            &spec(INPUTS_REF),
184        ],
185    ) {
186        git::Remote::Done(0, _) => adopt_both(),
187        // Origin did not answer; asking again would only double the wait.
188        git::Remote::TimedOut => (Mirror::TimedOut, Mirror::TimedOut),
189        _ => match git::remote(
190            root,
191            &[
192                "ls-remote",
193                "--exit-code",
194                "origin",
195                &full(NOTES_REF),
196                &full(INPUTS_REF),
197            ],
198        ) {
199            git::Remote::Done(0, listing) => {
200                let listed = |r: &str| {
201                    let f = full(r);
202                    listing
203                        .lines()
204                        .any(|l| l.split_whitespace().nth(1) == Some(f.as_str()))
205                };
206                let (has_main, has_inputs) = (listed(NOTES_REF), listed(INPUTS_REF));
207                let mut want = Vec::new();
208                if has_main {
209                    want.push(spec(NOTES_REF));
210                }
211                if has_inputs {
212                    want.push(spec(INPUTS_REF));
213                }
214                let fetched = if want.is_empty() {
215                    None
216                } else {
217                    let mut args = vec!["fetch", "--quiet", "origin"];
218                    args.extend(want.iter().map(String::as_str));
219                    Some(git::remote(root, &args))
220                };
221                let one = |present: bool, r: &str, old: Option<&str>, loud: &mut Vec<String>| {
222                    if !present {
223                        return drop_mirror(root, loud, r);
224                    }
225                    match &fetched {
226                        Some(git::Remote::Done(0, _)) => adopt(root, r, &tmp(r), old),
227                        Some(git::Remote::TimedOut) => Mirror::TimedOut,
228                        _ => Mirror::Unreachable,
229                    }
230                };
231                let m = one(has_main, NOTES_REF, old_main.as_deref(), loud);
232                let i = one(has_inputs, INPUTS_REF, old_inputs.as_deref(), loud);
233                (m, i)
234            }
235            git::Remote::Done(2, _) => (
236                drop_mirror(root, loud, NOTES_REF),
237                drop_mirror(root, loud, INPUTS_REF),
238            ),
239            git::Remote::TimedOut => (Mirror::TimedOut, Mirror::TimedOut),
240            _ => (Mirror::Unreachable, Mirror::Unreachable),
241        },
242    };
243    for r in [NOTES_REF, INPUTS_REF] {
244        let _ = git::succeeds_in(root, &["update-ref", "-d", &tmp(r)]);
245    }
246    states
247}
248
249// ---------------------------------------------------------------------------
250// Publishing (SPEC.md, "Producing")
251// ---------------------------------------------------------------------------
252
253/// How a publish is carried out. Every producer and every signed-notes ref
254/// (attestations, their inputs, provenance) goes through the same loop.
255pub struct Publish<'a> {
256    /// The remote to publish to.
257    pub remote: &'a str,
258    /// How many fetch-append-push rounds before a lost race is given up.
259    pub attempts: u32,
260    /// Wait between rounds, growing with the attempt (`sign.sh` in CI); a
261    /// hook on the commit path does not wait.
262    pub pause: bool,
263    /// The committer of the notes commit, for a runner with no identity of
264    /// its own; `None` uses the repository's.
265    pub identity: Option<(&'a str, &'a str)>,
266    /// Extra environment for the push itself (amont marks its own notes push
267    /// so its pre-push hook yields).
268    pub push_env: &'a [(&'a str, &'a str)],
269}
270
271/// What a publish came to.
272#[derive(Debug, Clone, PartialEq, Eq)]
273#[non_exhaustive]
274pub enum Published {
275    /// The remote now holds every block.
276    Pushed,
277    /// Every block was already there: nothing was pushed.
278    AlreadyPresent,
279    /// Written to the local ref only (see [`publish_local`]).
280    Local,
281    /// The push failed, after retries if the failure was a lost race.
282    PushFailed,
283    /// A note could not be written at all.
284    Error,
285}
286
287/// Why a push that exited non-zero failed, read from git's stderr.
288///
289/// A lost race is worth one more fetch-and-try; an unreachable remote says
290/// nothing about the push itself; everything else is a refusal to report
291/// verbatim. An unrecognised message is `Refused`, the direction that never
292/// claims a push was merely unlucky.
293#[derive(Debug, Clone, Copy, PartialEq, Eq)]
294#[non_exhaustive]
295pub enum PushFailure {
296    Raced,
297    Unreachable,
298    Refused,
299}
300
301/// The markers behind [`PushFailure`]. Raced wins over Unreachable: a
302/// message naming both is a race that happened on a reachable remote. A
303/// hook's own output rides the same stream, so Unreachable is read only from
304/// the lines git and ssh themselves print.
305pub fn classify_push(stderr: &str) -> PushFailure {
306    const RACED: [&str; 5] = [
307        "non-fast-forward",
308        "fetch first",
309        "stale info",
310        "cannot lock ref",
311        "failed to lock",
312    ];
313    const UNREACHABLE: [&str; 5] = [
314        "Could not resolve host",
315        "Connection refused",
316        "Connection timed out",
317        "unable to access",
318        "Could not read from remote repository",
319    ];
320    let own = |l: &&str| {
321        let l = l.trim_start();
322        l.starts_with("fatal:") || l.starts_with("ssh:") || l.starts_with("error:")
323    };
324    if RACED.iter().any(|m| stderr.contains(m)) {
325        PushFailure::Raced
326    } else if stderr
327        .lines()
328        .filter(own)
329        .any(|l| UNREACHABLE.iter().any(|m| l.contains(m)))
330    {
331        PushFailure::Unreachable
332    } else {
333        PushFailure::Refused
334    }
335}
336
337/// `git notes` writes a commit on the notes ref, and needs an identity.
338fn notes_git(root: &Path, identity: Option<(&str, &str)>) -> Command {
339    let mut cmd = Command::new("git");
340    cmd.current_dir(root).stdin(Stdio::null());
341    if let Some((name, email)) = identity {
342        cmd.args([
343            "-c",
344            &format!("user.name={name}"),
345            "-c",
346            &format!("user.email={email}"),
347        ]);
348    }
349    cmd
350}
351
352/// Append `block` to the note on `object` in `notes_ref`, unless that exact
353/// block is already there. `Some(true)` when it appended, `Some(false)` when
354/// the block was present, `None` when the note could not be written.
355fn append(
356    root: &Path,
357    identity: Option<(&str, &str)>,
358    notes_ref: &str,
359    object: &str,
360    block: &str,
361) -> Option<bool> {
362    let existing =
363        git::stdout_in(root, &["notes", "--ref", notes_ref, "show", object]).unwrap_or_default();
364    if existing.contains(block.trim_end()) {
365        return Some(false);
366    }
367    notes_git(root, identity)
368        .args(["notes", "--ref", notes_ref, "append", "-m", block, object])
369        .stdout(Stdio::null())
370        .stderr(Stdio::null())
371        .status()
372        .ok()?
373        .success()
374        .then_some(true)
375}
376
377/// Attach each `(object, block)` on the REMOTE's copy of `notes_ref` (short
378/// name, under `refs/notes/`) and push. Never `notes add -f`, which erases
379/// every other producer's block, and never a push of the local ref, which
380/// would re-publish a block origin revoked:
381///
382/// 1. fetch the remote's ref into a temporary ref;
383/// 2. append each block that is not already there (ed25519 signatures are
384///    deterministic, so a re-run produces the same bytes);
385/// 3. push the temporary ref; a lost race goes back to 1.
386///
387/// After a push to `origin` the local ref is set to what was published;
388/// another remote's content never enters it. The temporary ref is removed on
389/// every path. `log` gets one line per retry or refusal, credentials in a
390/// remote URL removed.
391pub fn publish(
392    root: &Path,
393    how: &Publish,
394    notes_ref: &str,
395    items: &[(String, String)],
396    log: &mut Vec<String>,
397) -> Published {
398    let full = format!("refs/notes/{notes_ref}");
399    let tref = format!("attest-publish-{}-{notes_ref}", std::process::id());
400    let tfull = format!("refs/notes/{tref}");
401    let mut attempt = 0;
402    let outcome = loop {
403        attempt += 1;
404        let _ = git::succeeds_in(root, &["update-ref", "-d", &tfull]);
405        // A remote with no such ref yet fails the fetch, harmlessly: append
406        // then creates the note from nothing.
407        let _ = git::succeeds_in(
408            root,
409            &["fetch", "--quiet", how.remote, &format!("+{full}:{tfull}")],
410        );
411        let mut appended = false;
412        for (object, block) in items {
413            match append(root, how.identity, &tref, object, block) {
414                Some(true) => appended = true,
415                Some(false) => {}
416                None => return cleanup(root, &tfull, Published::Error),
417            }
418        }
419        if !appended {
420            break Published::AlreadyPresent;
421        }
422        let out = Command::new("git")
423            .current_dir(root)
424            .args(["push", "--quiet", how.remote, &format!("{tfull}:{full}")])
425            .envs(how.push_env.iter().copied())
426            .stdin(Stdio::null())
427            .stdout(Stdio::null())
428            .output();
429        let Ok(out) = out else {
430            log.push(format!("push of {full} could not run git"));
431            break Published::PushFailed;
432        };
433        if out.status.success() {
434            if how.remote == "origin" {
435                let _ = git::succeeds_in(root, &["update-ref", &full, &tfull]);
436            }
437            break Published::Pushed;
438        }
439        let why = scrub(&String::from_utf8_lossy(&out.stderr));
440        match classify_push(&why) {
441            PushFailure::Raced if attempt < how.attempts => {
442                log.push(format!(
443                    "push of {full} rejected, another job attested this tree meanwhile; retrying"
444                ));
445                if how.pause {
446                    std::thread::sleep(std::time::Duration::from_secs(
447                        u64::from(attempt) + u64::from(std::process::id() % 3),
448                    ));
449                }
450            }
451            PushFailure::Raced => {
452                log.push(format!("gave up on {full} after {attempt} attempts: {why}"));
453                break Published::PushFailed;
454            }
455            _ => {
456                log.push(format!("push of {full} refused: {why}"));
457                break Published::PushFailed;
458            }
459        }
460    };
461    cleanup(root, &tfull, outcome)
462}
463
464fn cleanup(root: &Path, tfull: &str, outcome: Published) -> Published {
465    let _ = git::succeeds_in(root, &["update-ref", "-d", tfull]);
466    outcome
467}
468
469/// git's stderr on one line, with any `user:token@` in a remote URL removed.
470pub(crate) fn scrub(stderr: &str) -> String {
471    let mut out = String::new();
472    for word in stderr.split_whitespace() {
473        if !out.is_empty() {
474            out.push(' ');
475        }
476        match word.find("://") {
477            Some(i) => {
478                let (scheme, rest) = word.split_at(i + 3);
479                let host_start = match rest.find('/') {
480                    Some(slash) => rest[..slash].rfind('@').map(|a| a + 1).unwrap_or(0),
481                    None => rest.rfind('@').map(|a| a + 1).unwrap_or(0),
482                };
483                out.push_str(scheme);
484                out.push_str(&rest[host_start..]);
485            }
486            None => out.push_str(word),
487        }
488    }
489    out
490}
491
492/// The same, into the local `notes_ref` only, with no remote involved: what
493/// `sign --no-push` writes under `refs/notes/attest-local/`.
494pub fn publish_local(
495    root: &Path,
496    identity: Option<(&str, &str)>,
497    notes_ref: &str,
498    items: &[(String, String)],
499) -> Published {
500    let mut appended = false;
501    for (object, block) in items {
502        match append(root, identity, notes_ref, object, block) {
503            Some(true) => appended = true,
504            Some(false) => {}
505            None => return Published::Error,
506        }
507    }
508    if appended {
509        Published::Local
510    } else {
511        Published::AlreadyPresent
512    }
513}
git_attest/notes.rs

git_attest/
notes.rs
