git_attest/
notes.rs

1//! The notes refs as origin's mirror (SPEC.md, "Which refs are read").
2
3use std::path::Path;
4use std::process::{Command, Stdio};
5
6use crate::format::{Names, WRITE};
7use crate::git;
8
9/// What became of one notes ref. SPEC.md, "Lookup".
10///
11/// `refs/notes/{greengate,amont}-attest[-inputs]` are ORIGIN'S MIRROR, and origin is the
12/// only place an attestation can be revoked, so only a mirror that is
13/// origin's ref right now — or a repository with no origin at all, which is
14/// for fixtures and local use — is judged.
15#[derive(Clone, Copy, Debug, PartialEq)]
16pub enum Mirror {
17    /// Fetched: the mirror is origin's ref.
18    Fresh,
19    /// No remote named origin: the local ref is judged as it is.
20    NoOrigin,
21    /// Origin answered and has no such ref: the mirror was deleted.
22    Deleted,
23    /// As `Deleted`, but the delete failed: the stale mirror is not judged.
24    Undeletable,
25    /// Origin is configured and did not answer: not judged.
26    Unreachable,
27    /// As `Unreachable`, because the deadline cut the call off.
28    TimedOut,
29    /// Origin's copy was fetched but could not be made the local ref (a
30    /// stale lock, say): the stale local copy is not judged.
31    Unwritable,
32}
33
34impl Mirror {
35    pub fn as_str(self) -> &'static str {
36        match self {
37            Mirror::Fresh => "fetched",
38            Mirror::NoOrigin => "no-origin",
39            Mirror::Deleted => "absent",
40            Mirror::Undeletable => "undeletable",
41            Mirror::Unreachable | Mirror::TimedOut => "unreachable",
42            Mirror::Unwritable => "unwritable",
43        }
44    }
45    pub fn judged(self) -> bool {
46        matches!(self, Mirror::Fresh | Mirror::NoOrigin)
47    }
48}
49
50/// Delete the mirror of `notes_ref` because origin has none.
51fn drop_mirror(root: &Path, loud: &mut Vec<String>, notes_ref: &str) -> Mirror {
52    let full = format!("refs/notes/{notes_ref}");
53    let Some(oid) = git::stdout_in(root, &["rev-parse", "--verify", "--quiet", &full]) else {
54        return Mirror::Deleted;
55    };
56    if git::succeeds_in(root, &["update-ref", "-d", &full, &oid]) {
57        loud.push(format!(
58            "origin has no {full}; deleted the local mirror (was {oid}); restore: git update-ref {full} {oid}"
59        ));
60        Mirror::Deleted
61    } else {
62        Mirror::Undeletable
63    }
64}
65
66/// Where fetches land before they become the mirror: outside `refs/notes/`,
67/// one directory per process id, so no notes push or pruning fetch ever
68/// touches them.
69pub(crate) const TMP_NS: &str = "refs/attest-tmp/";
70
71/// Remove the throwaways of runs killed before their cleanup — only those
72/// whose process is gone, so a concurrent run in the same clone keeps its
73/// own. Where liveness cannot be asked, nothing is swept: a leftover outside
74/// `refs/notes/` is inert.
75fn sweep_tmp_refs(root: &Path) {
76    let Some(refs) = git::stdout_in(root, &["for-each-ref", "--format=%(refname)", TMP_NS]) else {
77        return;
78    };
79    let me = std::process::id().to_string();
80    for r in refs.lines() {
81        let Some(pid) = r
82            .strip_prefix(TMP_NS)
83            .and_then(|rest| rest.split('/').next())
84            .filter(|p| !p.is_empty() && p.bytes().all(|b| b.is_ascii_digit()))
85        else {
86            continue;
87        };
88        if pid != me && process_gone(pid) {
89            let _ = git::succeeds_in(root, &["update-ref", "-d", r]);
90        }
91    }
92}
93
94/// True only on a positive "no such process" (read in the C locale); any
95/// other answer, or none, counts as alive, so nothing live is swept.
96fn process_gone(pid: &str) -> bool {
97    if !cfg!(unix) {
98        return false;
99    }
100    Command::new("kill")
101        .args(["-0", pid])
102        .env("LC_ALL", "C")
103        .stdin(Stdio::null())
104        .stdout(Stdio::null())
105        .stderr(Stdio::piped())
106        .output()
107        .is_ok_and(|o| {
108            !o.status.success()
109                && String::from_utf8_lossy(&o.stderr)
110                    .to_lowercase()
111                    .contains("no such process")
112        })
113}
114
115/// Make the fetched copy in `tmp` the mirror of `notes_ref`, by a
116/// compare-and-swap against `old` (what was there before the fetch). A kill
117/// can only ever half-write the throwaway, never the mirror. When the swap
118/// fails, the mirror is judged only if it now holds EXACTLY origin's copy (a
119/// concurrent run got there first, or origin is unchanged under a lock).
120fn adopt(root: &Path, notes_ref: &str, tmp: &str, old: Option<&str>) -> Mirror {
121    let full = format!("refs/notes/{notes_ref}");
122    let Some(new) = git::stdout_in(root, &["rev-parse", "--verify", "--quiet", tmp]) else {
123        return Mirror::Unwritable;
124    };
125    if git::succeeds_in(root, &["update-ref", &full, &new, old.unwrap_or("")]) {
126        return Mirror::Fresh;
127    }
128    match git::stdout_in(root, &["rev-parse", "--verify", "--quiet", &full]) {
129        Some(now) if now == new => Mirror::Fresh,
130        _ => Mirror::Unwritable,
131    }
132}
133
134/// Bring both mirrors of the written names ([`WRITE`]) in line with origin:
135/// one fetch of both refs into
136/// throwaways and, when that fails without timing out, one `ls-remote` to
137/// tell "origin has no such ref" from "origin did not answer" — then, if some
138/// refs exist, one more fetch of those. A silent origin costs one call (15 s
139/// at most), a dead one two (30 s), the worst case three (45 s).
140pub fn sync_mirrors(root: &Path, loud: &mut Vec<String>) -> (Mirror, Mirror) {
141    let synced = sync_mirrors_of(root, &WRITE);
142    loud.extend(synced.loud);
143    (synced.notes, synced.inputs)
144}
145
146/// What one name set's sync came to: the state of its notes ref and of its
147/// inputs ref, and the lines that report a change to the repository's refs
148/// (a mirror deleted, with how to restore it).
149#[derive(Debug, Clone, PartialEq)]
150#[non_exhaustive]
151pub struct Synced {
152    pub notes: Mirror,
153    pub inputs: Mirror,
154    pub loud: Vec<String>,
155}
156
157impl Synced {
158    /// How origin failed to answer for this set, if it did not: `TimedOut`
159    /// when a deadline cut any call off, `Unreachable` when both refs are.
160    /// Another set's sync would only wait again, so it takes this state.
161    pub fn origin_silent(&self) -> Option<Mirror> {
162        if self.notes == Mirror::TimedOut || self.inputs == Mirror::TimedOut {
163            Some(Mirror::TimedOut)
164        } else if self.notes == Mirror::Unreachable && self.inputs == Mirror::Unreachable {
165            Some(Mirror::Unreachable)
166        } else {
167            None
168        }
169    }
170}
171
172/// [`sync_mirrors`] for one name set's two refs. A verifier syncs each set on
173/// its own, so a set origin does not have (the old one after the census, the
174/// new one before the writers switch) is `Deleted`, which judges nothing and
175/// changes no verdict, never `Unreachable`.
176pub fn sync_mirrors_of(root: &Path, names: &Names) -> Synced {
177    // Every line this sync has to say goes into `loud`, returned with the
178    // mirrors: the helpers below borrow it, nothing outside does.
179    let mut loud = Vec::new();
180    if !git::succeeds_in(root, &["remote", "get-url", "origin"]) {
181        return Synced {
182            notes: Mirror::NoOrigin,
183            inputs: Mirror::NoOrigin,
184            loud,
185        };
186    }
187    sweep_tmp_refs(root);
188    let pid = std::process::id();
189    let full = |r: &str| format!("refs/notes/{r}");
190    let tmp = |r: &str| format!("{TMP_NS}{pid}/{r}");
191    let spec = |r: &str| format!("+{}:{}", full(r), tmp(r));
192    // Our own throwaways, and any lock a dead run with our pid left on them.
193    // Known limits: a remove that fails leaves the fetch to fail and read as
194    // unreachable (safe: nothing judged); a killed fetch's lock with no ref
195    // beside it is not swept, only replaced when its pid comes round again.
196    for r in [names.notes_ref, names.inputs_ref] {
197        let _ = git::succeeds_in(root, &["update-ref", "-d", &tmp(r)]);
198        if let Some(lock) = git::stdout_in(
199            root,
200            &["rev-parse", "--git-path", &format!("{}.lock", tmp(r))],
201        ) {
202            // `--git-path` answers relative to where git ran.
203            let _ = std::fs::remove_file(root.join(lock));
204        }
205    }
206    let old_main = git::stdout_in(
207        root,
208        &["rev-parse", "--verify", "--quiet", &full(names.notes_ref)],
209    );
210    let old_inputs = git::stdout_in(
211        root,
212        &["rev-parse", "--verify", "--quiet", &full(names.inputs_ref)],
213    );
214    let adopt_both = || {
215        (
216            adopt(
217                root,
218                names.notes_ref,
219                &tmp(names.notes_ref),
220                old_main.as_deref(),
221            ),
222            adopt(
223                root,
224                names.inputs_ref,
225                &tmp(names.inputs_ref),
226                old_inputs.as_deref(),
227            ),
228        )
229    };
230    let states = match git::remote(
231        root,
232        &[
233            "fetch",
234            "--quiet",
235            "origin",
236            &spec(names.notes_ref),
237            &spec(names.inputs_ref),
238        ],
239    ) {
240        git::Remote::Done(0, _) => adopt_both(),
241        // Origin did not answer; asking again would only double the wait.
242        git::Remote::TimedOut => (Mirror::TimedOut, Mirror::TimedOut),
243        _ => match git::remote(
244            root,
245            &[
246                "ls-remote",
247                "--exit-code",
248                "origin",
249                &full(names.notes_ref),
250                &full(names.inputs_ref),
251            ],
252        ) {
253            git::Remote::Done(0, listing) => {
254                let listed = |r: &str| {
255                    let f = full(r);
256                    listing
257                        .lines()
258                        .any(|l| l.split_whitespace().nth(1) == Some(f.as_str()))
259                };
260                let (has_main, has_inputs) = (listed(names.notes_ref), listed(names.inputs_ref));
261                let mut want = Vec::new();
262                if has_main {
263                    want.push(spec(names.notes_ref));
264                }
265                if has_inputs {
266                    want.push(spec(names.inputs_ref));
267                }
268                let fetched = if want.is_empty() {
269                    None
270                } else {
271                    let mut args = vec!["fetch", "--quiet", "origin"];
272                    args.extend(want.iter().map(String::as_str));
273                    Some(git::remote(root, &args))
274                };
275                let one = |present: bool, r: &str, old: Option<&str>, loud: &mut Vec<String>| {
276                    if !present {
277                        return drop_mirror(root, loud, r);
278                    }
279                    match &fetched {
280                        Some(git::Remote::Done(0, _)) => adopt(root, r, &tmp(r), old),
281                        Some(git::Remote::TimedOut) => Mirror::TimedOut,
282                        _ => Mirror::Unreachable,
283                    }
284                };
285                let notes_mirror = one(has_main, names.notes_ref, old_main.as_deref(), &mut loud);
286                let inputs_mirror = one(
287                    has_inputs,
288                    names.inputs_ref,
289                    old_inputs.as_deref(),
290                    &mut loud,
291                );
292                (notes_mirror, inputs_mirror)
293            }
294            git::Remote::Done(2, _) => (
295                drop_mirror(root, &mut loud, names.notes_ref),
296                drop_mirror(root, &mut loud, names.inputs_ref),
297            ),
298            git::Remote::TimedOut => (Mirror::TimedOut, Mirror::TimedOut),
299            _ => (Mirror::Unreachable, Mirror::Unreachable),
300        },
301    };
302    for r in [names.notes_ref, names.inputs_ref] {
303        let _ = git::succeeds_in(root, &["update-ref", "-d", &tmp(r)]);
304    }
305    let (notes, inputs) = states;
306    Synced {
307        notes,
308        inputs,
309        loud,
310    }
311}
312
313// ---------------------------------------------------------------------------
314// Publishing (SPEC.md, "Producing")
315// ---------------------------------------------------------------------------
316
317/// How a publish is carried out. Every producer and every signed-notes ref
318/// (attestations, their inputs, provenance) goes through the same loop.
319pub struct Publish<'a> {
320    /// The remote to publish to.
321    pub remote: &'a str,
322    /// How many fetch-append-push rounds before a lost race is given up.
323    pub attempts: u32,
324    /// Wait between rounds, growing with the attempt (`sign.sh` in CI); a
325    /// hook on the commit path does not wait.
326    pub pause: bool,
327    /// The committer of the notes commit, for a runner with no identity of
328    /// its own; `None` uses the repository's.
329    pub identity: Option<(&'a str, &'a str)>,
330    /// Extra environment for the push itself (amont marks its own notes push
331    /// so its pre-push hook yields).
332    pub push_env: &'a [(&'a str, &'a str)],
333}
334
335/// What a publish came to.
336#[derive(Debug, Clone, PartialEq, Eq)]
337#[non_exhaustive]
338pub enum Published {
339    /// The remote now holds every block.
340    Pushed,
341    /// Every block was already there: nothing was pushed.
342    AlreadyPresent,
343    /// Written to the local ref only (see [`publish_local`]).
344    Local,
345    /// The push failed, after retries if the failure was a lost race.
346    PushFailed,
347    /// A note could not be written at all.
348    Error,
349}
350
351/// Why a push that exited non-zero failed, read from git's stderr.
352///
353/// A lost race is worth one more fetch-and-try; an unreachable remote says
354/// nothing about the push itself; everything else is a refusal to report
355/// verbatim. An unrecognised message is `Refused`, the direction that never
356/// claims a push was merely unlucky.
357#[derive(Debug, Clone, Copy, PartialEq, Eq)]
358#[non_exhaustive]
359pub enum PushFailure {
360    Raced,
361    Unreachable,
362    Refused,
363}
364
365/// The markers behind [`PushFailure`]. Raced wins over Unreachable: a
366/// message naming both is a race that happened on a reachable remote. A
367/// hook's own output rides the same stream, so Unreachable is read only from
368/// the lines git and ssh themselves print.
369pub fn classify_push(stderr: &str) -> PushFailure {
370    const RACED: [&str; 5] = [
371        "non-fast-forward",
372        "fetch first",
373        "stale info",
374        "cannot lock ref",
375        "failed to lock",
376    ];
377    const UNREACHABLE: [&str; 5] = [
378        "Could not resolve host",
379        "Connection refused",
380        "Connection timed out",
381        "unable to access",
382        "Could not read from remote repository",
383    ];
384    let own = |l: &&str| {
385        let l = l.trim_start();
386        l.starts_with("fatal:") || l.starts_with("ssh:") || l.starts_with("error:")
387    };
388    if RACED.iter().any(|m| stderr.contains(m)) {
389        PushFailure::Raced
390    } else if stderr
391        .lines()
392        .filter(own)
393        .any(|l| UNREACHABLE.iter().any(|m| l.contains(m)))
394    {
395        PushFailure::Unreachable
396    } else {
397        PushFailure::Refused
398    }
399}
400
401/// `git notes` writes a commit on the notes ref, and needs an identity.
402fn notes_git(root: &Path, identity: Option<(&str, &str)>) -> Command {
403    let mut cmd = Command::new("git");
404    cmd.current_dir(root).stdin(Stdio::null());
405    if let Some((name, email)) = identity {
406        cmd.args([
407            "-c",
408            &format!("user.name={name}"),
409            "-c",
410            &format!("user.email={email}"),
411        ]);
412    }
413    cmd
414}
415
416/// Append `block` to the note on `object` in `notes_ref`, unless that exact
417/// block is already there. `Some(true)` when it appended, `Some(false)` when
418/// the block was present, `None` when the note could not be written.
419fn append(
420    root: &Path,
421    identity: Option<(&str, &str)>,
422    notes_ref: &str,
423    object: &str,
424    block: &str,
425) -> Option<bool> {
426    let existing =
427        git::stdout_in(root, &["notes", "--ref", notes_ref, "show", object]).unwrap_or_default();
428    if existing.contains(block.trim_end()) {
429        return Some(false);
430    }
431    notes_git(root, identity)
432        .args(["notes", "--ref", notes_ref, "append", "-m", block, object])
433        .stdout(Stdio::null())
434        .stderr(Stdio::null())
435        .status()
436        .ok()?
437        .success()
438        .then_some(true)
439}
440
441/// Attach each `(object, block)` on the REMOTE's copy of `notes_ref` (short
442/// name, under `refs/notes/`) and push. Never `notes add -f`, which erases
443/// every other producer's block, and never a push of the local ref, which
444/// would re-publish a block origin revoked:
445///
446/// 1. fetch the remote's ref into a temporary ref;
447/// 2. append each block that is not already there (ed25519 signatures are
448///    deterministic, so a re-run produces the same bytes);
449/// 3. push the temporary ref; a lost race goes back to 1.
450///
451/// After a push to `origin` the local ref is set to what was published;
452/// another remote's content never enters it. The temporary ref is removed on
453/// every path. `log` gets one line per retry or refusal, credentials in a
454/// remote URL removed.
455pub fn publish(
456    root: &Path,
457    how: &Publish,
458    notes_ref: &str,
459    items: &[(String, String)],
460    log: &mut Vec<String>,
461) -> Published {
462    let full = format!("refs/notes/{notes_ref}");
463    let tref = format!("attest-publish-{}-{notes_ref}", std::process::id());
464    let tfull = format!("refs/notes/{tref}");
465    let mut attempt = 0;
466    let outcome = loop {
467        attempt += 1;
468        let _ = git::succeeds_in(root, &["update-ref", "-d", &tfull]);
469        // A remote with no such ref yet fails the fetch, harmlessly: append
470        // then creates the note from nothing.
471        let _ = git::succeeds_in(
472            root,
473            &["fetch", "--quiet", how.remote, &format!("+{full}:{tfull}")],
474        );
475        let mut appended = false;
476        for (object, block) in items {
477            match append(root, how.identity, &tref, object, block) {
478                Some(true) => appended = true,
479                Some(false) => {}
480                None => return cleanup(root, &tfull, Published::Error),
481            }
482        }
483        if !appended {
484            break Published::AlreadyPresent;
485        }
486        let out = Command::new("git")
487            .current_dir(root)
488            .args(["push", "--quiet", how.remote, &format!("{tfull}:{full}")])
489            .envs(how.push_env.iter().copied())
490            .stdin(Stdio::null())
491            .stdout(Stdio::null())
492            .output();
493        let Ok(out) = out else {
494            log.push(format!("push of {full} could not run git"));
495            break Published::PushFailed;
496        };
497        if out.status.success() {
498            if how.remote == "origin" {
499                let _ = git::succeeds_in(root, &["update-ref", &full, &tfull]);
500            }
501            break Published::Pushed;
502        }
503        let why = scrub(&String::from_utf8_lossy(&out.stderr));
504        match classify_push(&why) {
505            PushFailure::Raced if attempt < how.attempts => {
506                log.push(format!(
507                    "push of {full} rejected, another job attested this tree meanwhile; retrying"
508                ));
509                if how.pause {
510                    std::thread::sleep(std::time::Duration::from_secs(
511                        u64::from(attempt) + u64::from(std::process::id() % 3),
512                    ));
513                }
514            }
515            PushFailure::Raced => {
516                log.push(format!("gave up on {full} after {attempt} attempts: {why}"));
517                break Published::PushFailed;
518            }
519            _ => {
520                log.push(format!("push of {full} refused: {why}"));
521                break Published::PushFailed;
522            }
523        }
524    };
525    cleanup(root, &tfull, outcome)
526}
527
528fn cleanup(root: &Path, tfull: &str, outcome: Published) -> Published {
529    let _ = git::succeeds_in(root, &["update-ref", "-d", tfull]);
530    outcome
531}
532
533/// git's stderr on one line, with any `user:token@` in a remote URL removed.
534pub(crate) fn scrub(stderr: &str) -> String {
535    let mut out = String::new();
536    for word in stderr.split_whitespace() {
537        if !out.is_empty() {
538            out.push(' ');
539        }
540        match word.find("://") {
541            Some(i) => {
542                let (scheme, rest) = word.split_at(i + 3);
543                let host_start = match rest.find('/') {
544                    Some(slash) => rest[..slash].rfind('@').map(|a| a + 1).unwrap_or(0),
545                    None => rest.rfind('@').map(|a| a + 1).unwrap_or(0),
546                };
547                out.push_str(scheme);
548                out.push_str(&rest[host_start..]);
549            }
550            None => out.push_str(word),
551        }
552    }
553    out
554}
555
556/// The same, into the local `notes_ref` only, with no remote involved: what
557/// `sign --no-push` writes under `refs/notes/attest-local/`.
558pub fn publish_local(
559    root: &Path,
560    identity: Option<(&str, &str)>,
561    notes_ref: &str,
562    items: &[(String, String)],
563) -> Published {
564    let mut appended = false;
565    for (object, block) in items {
566        match append(root, identity, notes_ref, object, block) {
567            Some(true) => appended = true,
568            Some(false) => {}
569            None => return Published::Error,
570        }
571    }
572    if appended {
573        Published::Local
574    } else {
575        Published::AlreadyPresent
576    }
577}
git_attest/notes.rs

git_attest/
notes.rs
