Skip to main content

jouet_git/
notes.rs

1//! Lines of text kept as git notes on commits, under one notes ref, the way
2//! tak keeps its measurements:
3//!
4//! - The data is about a commit, which is what notes are for, and it never
5//!   touches the commits or the work tree.
6//! - Concurrent writers merge with git's `cat_sort_uniq` strategy, so each
7//!   note is a set of lines, kept sorted and without duplicates locally too,
8//!   and no merge code is needed.
9//! - A push is never forced. tak learned why: a forced push from a checkout
10//!   that had not fetched the ref replaced all of its history at once. A
11//!   rejected push fetches, merges and tries again instead.
12
13use crate::{Error, Git};
14use jouet_output::exit::Exit;
15use std::collections::HashMap;
16use std::io::{BufRead, BufReader, Read, Write};
17use std::process::Stdio;
18
19/// How many times a rejected push fetches, merges and tries again.
20const PUSH_ATTEMPTS: u32 = 5;
21
22/// The notes under `refs/notes/NAME` of one repository.
23pub struct Notes {
24    git: Git,
25    name: String,
26}
27
28impl Notes {
29    pub fn new(git: Git, name: &str) -> Self {
30        Self {
31            git,
32            name: name.to_string(),
33        }
34    }
35
36    fn reference(&self) -> String {
37        format!("refs/notes/{}", self.name)
38    }
39
40    /// Where a fetch puts the remote's notes before they are merged, so it
41    /// never overwrites lines not yet pushed.
42    fn remote_reference(&self) -> String {
43        format!("refs/notes/{}-remote", self.name)
44    }
45
46    /// Adds `lines` to the note on `commit`, keeping what it holds.
47    pub fn append(&self, commit: &str, lines: &[String]) -> Result<(), Error> {
48        if lines.is_empty() {
49            return Ok(());
50        }
51        let reference = self.reference();
52        let mut all: Vec<String> = self
53            .git
54            .line(&["notes", "--ref", &reference, "show", commit])
55            .map(|note| note.lines().map(str::to_string).collect())
56            .unwrap_or_default();
57        all.extend(lines.iter().cloned());
58        // As cat_sort_uniq would leave them, so a local write and a merge
59        // of the same lines agree.
60        all.sort();
61        all.dedup();
62        let note = all.join("\n");
63        self.committing(&[
64            "notes", "--ref", &reference, "add", "-f", "-m", &note, commit,
65        ])
66    }
67
68    /// The notes on `rev` and its first-parent ancestors, newest first, up
69    /// to `depth` commits back: each commit that has a note, with its lines.
70    pub fn along(&self, rev: &str, depth: usize) -> Vec<(String, Vec<String>)> {
71        let reference = self.reference();
72        let Some(listed) = self.git.line(&["notes", "--ref", &reference, "list"]) else {
73            return Vec::new();
74        };
75        let blobs: HashMap<&str, &str> = listed
76            .lines()
77            .filter_map(|l| l.split_once(' '))
78            .map(|(blob, commit)| (commit, blob))
79            .collect();
80        let depth = depth.to_string();
81        let Some(commits) = self
82            .git
83            .line(&["rev-list", "--first-parent", "-n", &depth, rev, "--"])
84        else {
85            return Vec::new();
86        };
87        let noted: Vec<(&str, &str)> = commits
88            .lines()
89            .filter_map(|c| blobs.get(c).map(|b| (c, *b)))
90            .collect();
91        let texts = self.blobs(noted.iter().map(|(_, b)| *b));
92        noted
93            .into_iter()
94            .zip(texts)
95            .map(|((commit, _), text)| {
96                (
97                    commit.to_string(),
98                    text.lines().map(str::to_string).collect(),
99                )
100            })
101            .collect()
102    }
103
104    /// The text of each blob, in order, through one `git cat-file --batch`.
105    fn blobs<'a>(&self, ids: impl Iterator<Item = &'a str>) -> Vec<String> {
106        let ids: Vec<&str> = ids.collect();
107        if ids.is_empty() {
108            return Vec::new();
109        }
110        let mut cmd = self.git.command(&["cat-file", "--batch"]);
111        let Ok(mut child) = cmd
112            .stdin(Stdio::piped())
113            .stdout(Stdio::piped())
114            .stderr(Stdio::null())
115            .spawn()
116        else {
117            return Vec::new();
118        };
119        let input: String = ids.iter().map(|id| format!("{id}\n")).collect();
120        let mut stdin = child.stdin.take().expect("stdin is piped");
121        let writer = std::thread::spawn(move || stdin.write_all(input.as_bytes()));
122        let mut out = BufReader::new(child.stdout.take().expect("stdout is piped"));
123        let mut texts = Vec::with_capacity(ids.len());
124        for _ in &ids {
125            let mut header = String::new();
126            if out.read_line(&mut header).unwrap_or(0) == 0 {
127                break;
128            }
129            // `<id> blob <size>`, or `<id> missing`.
130            let Some(size) = header
131                .split_whitespace()
132                .nth(2)
133                .and_then(|s| s.parse::<usize>().ok())
134            else {
135                texts.push(String::new());
136                continue;
137            };
138            let mut body = vec![0; size + 1];
139            if out.read_exact(&mut body).is_err() {
140                break;
141            }
142            body.pop();
143            texts.push(String::from_utf8_lossy(&body).into_owned());
144        }
145        let _ = writer.join();
146        let _ = child.wait();
147        texts.resize(ids.len(), String::new());
148        texts
149    }
150
151    /// Fetches `remote`'s notes and merges them into the local ones.
152    /// `Ok(false)` when the remote has none; an error when it cannot be
153    /// reached or read.
154    pub fn fetch(&self, remote: &str) -> Result<bool, Error> {
155        let reference = self.reference();
156        let listed = self
157            .git
158            .command(&["ls-remote", "--exit-code", remote, &reference])
159            .output()
160            .map_err(|e| Error::new(Exit::Failed, format!("running git: {e}")))?;
161        match listed.status.code() {
162            Some(0) => {}
163            // --exit-code: the remote answered, without the ref.
164            Some(2) => return Ok(false),
165            _ => {
166                return Err(Error::new(
167                    Exit::Failed,
168                    format!(
169                        "reading {reference} from {remote}: {}",
170                        String::from_utf8_lossy(&listed.stderr).trim()
171                    ),
172                ));
173            }
174        }
175        let spec = format!("+{reference}:{}", self.remote_reference());
176        self.git.text(&["fetch", "--quiet", remote, &spec])?;
177        self.absorb()?;
178        Ok(true)
179    }
180
181    /// Merges the fetched notes into the local ones, keeping both sides.
182    fn absorb(&self) -> Result<(), Error> {
183        let (local, remote) = (self.reference(), self.remote_reference());
184        if self.git.commit_sha(&local).is_none() {
185            // Nothing local yet: `notes merge` needs a ref to merge into.
186            return self.git.text(&["update-ref", &local, &remote]).map(drop);
187        }
188        self.committing(&[
189            "notes",
190            "--ref",
191            &local,
192            "merge",
193            "-q",
194            "-s",
195            "cat_sort_uniq",
196            &remote,
197        ])
198    }
199
200    /// Pushes the notes to `remote`, merging in what another writer pushed
201    /// first, and never forcing.
202    pub fn push(&self, remote: &str) -> Result<(), Error> {
203        let reference = self.reference();
204        if self.git.commit_sha(&reference).is_none() {
205            return Err(Error::new(
206                Exit::Precondition,
207                format!("there are no notes under {reference} to push"),
208            ));
209        }
210        let spec = format!("{reference}:{reference}");
211        let mut last = String::new();
212        for _ in 0..PUSH_ATTEMPTS {
213            let out = self
214                .git
215                .command(&["push", "--quiet", remote, &spec])
216                .output()
217                .map_err(|e| Error::new(Exit::Failed, format!("running git: {e}")))?;
218            if out.status.success() {
219                return Ok(());
220            }
221            last = String::from_utf8_lossy(&out.stderr).trim().to_string();
222            self.fetch(remote)?;
223        }
224        Err(Error::new(
225            Exit::Failed,
226            format!("could not push {reference} after {PUSH_ATTEMPTS} attempts: {last}"),
227        ))
228    }
229
230    /// A git command that writes a notes commit, which needs an author: CI
231    /// images often have none, so jouet's own stands in only then.
232    fn committing(&self, args: &[&str]) -> Result<(), Error> {
233        let mut all: Vec<&str> = Vec::new();
234        if self.git.line(&["config", "user.name"]).is_none() {
235            all.extend(["-c", "user.name=jouet"]);
236        }
237        if self.git.line(&["config", "user.email"]).is_none() {
238            all.extend(["-c", "user.email=jouet@localhost"]);
239        }
240        all.extend(args);
241        self.git.text(&all).map(drop)
242    }
243}
244
245#[cfg(test)]
246mod tests {
247    use super::*;
248    use std::path::Path;
249    use std::process::Command;
250
251    fn git(dir: &Path, args: &[&str]) -> String {
252        let out = Command::new("git")
253            .arg("-C")
254            .arg(dir)
255            .args(["-c", "user.name=t", "-c", "user.email=t@t"])
256            .args(args)
257            .output()
258            .unwrap();
259        assert!(
260            out.status.success(),
261            "git {args:?}: {}",
262            String::from_utf8_lossy(&out.stderr)
263        );
264        String::from_utf8_lossy(&out.stdout).trim().to_string()
265    }
266
267    fn repo(dir: &Path, commits: usize) -> Vec<String> {
268        git(dir, &["init", "-q", "-b", "main"]);
269        (0..commits)
270            .map(|i| {
271                git(
272                    dir,
273                    &["commit", "-q", "--allow-empty", "-m", &format!("c{i}")],
274                );
275                git(dir, &["rev-parse", "HEAD"])
276            })
277            .collect()
278    }
279
280    #[test]
281    fn appended_lines_are_a_sorted_set_read_back_newest_first() {
282        let dir = tempfile::tempdir().unwrap();
283        let shas = repo(dir.path(), 3);
284        let notes = Notes::new(Git::new(dir.path()), "t");
285        notes.append(&shas[0], &["b".into(), "a".into()]).unwrap();
286        notes.append(&shas[0], &["a".into(), "c".into()]).unwrap();
287        notes.append(&shas[2], &["z".into()]).unwrap();
288        let along = notes.along("HEAD", 10);
289        assert_eq!(
290            along,
291            [
292                (shas[2].clone(), vec!["z".to_string()]),
293                (shas[0].clone(), vec!["a".into(), "b".into(), "c".into()]),
294            ]
295        );
296        // Only `depth` commits back.
297        assert_eq!(notes.along("HEAD", 2).len(), 1);
298        // Another notes ref is someone else's.
299        assert!(
300            Notes::new(Git::new(dir.path()), "other")
301                .along("HEAD", 10)
302                .is_empty()
303        );
304    }
305
306    #[test]
307    fn a_fetch_says_whether_the_remote_has_notes_or_failed() {
308        let remote = tempfile::tempdir().unwrap();
309        git(remote.path(), &["init", "-q", "--bare"]);
310        let local = tempfile::tempdir().unwrap();
311        repo(local.path(), 1);
312        let notes = Notes::new(Git::new(local.path()), "t");
313        assert!(!notes.fetch(remote.path().to_str().unwrap()).unwrap());
314        let gone = local.path().join("no-such-remote");
315        assert!(notes.fetch(gone.to_str().unwrap()).is_err());
316    }
317
318    #[test]
319    fn a_push_merges_what_another_writer_pushed_first() {
320        let remote = tempfile::tempdir().unwrap();
321        git(remote.path(), &["init", "-q", "--bare"]);
322        let a = tempfile::tempdir().unwrap();
323        let shas = repo(a.path(), 1);
324        let url = remote.path().to_str().unwrap();
325        git(a.path(), &["push", "-q", url, "main"]);
326        let b = tempfile::tempdir().unwrap();
327        git(b.path(), &["clone", "-q", url, "."]);
328
329        let on_a = Notes::new(Git::new(a.path()), "t");
330        on_a.append(&shas[0], &["from a".into()]).unwrap();
331        on_a.push(url).unwrap();
332        // b never fetched a's notes: its push is rejected, merges, retries.
333        let on_b = Notes::new(Git::new(b.path()), "t");
334        on_b.append(&shas[0], &["from b".into()]).unwrap();
335        on_b.push(url).unwrap();
336
337        assert!(on_a.fetch(url).unwrap());
338        assert_eq!(
339            on_a.along("HEAD", 1),
340            [(shas[0].clone(), vec!["from a".to_string(), "from b".into()])]
341        );
342    }
343}