Skip to main content

magi/
release_local.rs

1//! Releasing without GitHub Actions (`[release] mode = "local"`).
2//!
3//! On a repository whose Actions never run, `auto-tag.yml` and `release.yml`
4//! never fire, so nothing tags or publishes after the release pull request
5//! merges. In local mode [`crate::release_watch`] merges that pull request on
6//! the owner's approval and then drives the [`Job`] defined here, from a clean
7//! detached checkout of the merge commit:
8//!
9//! 1. create and push the `vX.Y.Z` tag - **never over a tag that exists**
10//!    ([`tag_step`] is the whole decision);
11//! 2. run `[release] commands` one at a time, stopping at the first failure.
12//!
13//! - **Progress is saved before every step**, through the caller's `save`. A
14//!   step that was started and never finished (a crash, a kill) is
15//!   [`Job::interrupted`]: magi cannot know whether `gh release create` got as
16//!   far as the forge, so it holds for the owner instead of repeating it.
17//! - **A failure is a hold, not a retry.** [`run_job`] returns the reason; the
18//!   watcher records it, raises a notice and asks. Resuming skips the tag and
19//!   every command that already succeeded.
20//! - **Environment, not templates.** Commands see `MAGI_RELEASE_VERSION`,
21//!   `MAGI_RELEASE_TAG`, `MAGI_RELEASE_COMMIT` and `MAGI_RELEASE_PR`; teravars
22//!   renders the whole config before any release exists, so `{{ version }}`
23//!   cannot work.
24//! - Only `git` and the commands the operator wrote run; there is no API path.
25
26use std::path::{Path, PathBuf};
27use std::time::Duration;
28
29use anyhow::{Context, Result, bail};
30use serde::{Deserialize, Serialize};
31
32use crate::config::Release;
33use crate::git;
34use crate::proc::Quiet as _;
35
36/// Bytes of a command's output kept in the record; the whole of it goes to
37/// `release-<n>.out` beside the job.
38const TAIL: usize = 4_000;
39
40/// Head of a release branch name, before the version.
41pub const BRANCH_PREFIX: &str = "chore/release-v";
42
43/// What happened in one step, kept in the record.
44#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
45#[serde(default)]
46pub struct StepLog {
47    /// `tag` or the command line.
48    pub name: String,
49    /// Exit code; `None` for a signal or a timeout.
50    pub code: Option<i32>,
51    /// Last bytes of the combined output.
52    pub tail: String,
53}
54
55/// One release of one merged pull request, resumable.
56#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
57#[serde(default)]
58pub struct Job {
59    /// `X.Y.Z`, without the `v`.
60    pub version: String,
61    /// The merged release pull request.
62    pub pr_url: String,
63    /// The commit the pull request merged as; what the tag points at.
64    pub commit: String,
65    /// The tag exists on the remote at `commit`.
66    pub tag_done: bool,
67    /// Commands that finished successfully.
68    pub done: usize,
69    /// The step started and not yet finished (`tag` or `command N`).
70    pub running: Option<String>,
71    /// Why the job stopped. Held until the owner decides.
72    pub failed: Option<String>,
73    /// Everything ran.
74    pub finished: bool,
75    /// What each step that ran said, in order.
76    pub log: Vec<StepLog>,
77}
78
79impl Job {
80    /// A job that has not started.
81    pub fn new(version: &str, pr_url: &str, commit: &str) -> Self {
82        Self {
83            version: version.to_owned(),
84            pr_url: pr_url.to_owned(),
85            commit: commit.to_owned(),
86            ..Self::default()
87        }
88    }
89
90    /// The tag name, `vX.Y.Z`.
91    pub fn tag(&self) -> String {
92        format!("v{}", self.version)
93    }
94
95    /// Stopped in the middle of a step with nothing recorded about how it
96    /// ended: the outcome is unknown, so it is not retried on its own.
97    pub fn interrupted(&self) -> bool {
98        self.running.is_some() && self.failed.is_none() && !self.finished
99    }
100
101    /// Owner chose to retry: forget the stop and resume from the progress.
102    pub fn resume(&mut self) {
103        self.failed = None;
104        self.running = None;
105    }
106}
107
108/// `1.2.3` out of `chore/release-v1.2.3`.
109pub fn version_from_branch(branch: &str) -> Option<String> {
110    let v = branch.strip_prefix(BRANCH_PREFIX)?;
111    (!v.is_empty()).then(|| v.to_owned())
112}
113
114/// What to do about the tag. Pure.
115#[derive(Debug, Clone, PartialEq, Eq)]
116pub enum TagStep {
117    /// Neither the remote nor this checkout has it: create it and push.
118    Create,
119    /// A local tag already at the commit but not on the remote: push it.
120    PushExisting,
121    /// The remote already has it at the commit: nothing to do.
122    Present,
123    /// A tag of that name points elsewhere. Never touched.
124    Foreign(String),
125    /// The remote could not be read, so "does not exist" is not known.
126    Unreadable(String),
127}
128
129/// Decide the tag step from what was read. `remote` is the commit the remote's
130/// tag peels to (`Ok(None)`: absent), or the reason it could not be read;
131/// `local` is the commit of a local tag of that name.
132pub fn tag_step(
133    tag: &str,
134    commit: &str,
135    remote: std::result::Result<Option<&str>, &str>,
136    local: Option<&str>,
137) -> TagStep {
138    match remote {
139        Err(e) => TagStep::Unreadable(format!("could not read the remote's {tag}: {e}")),
140        Ok(Some(r)) if r == commit => TagStep::Present,
141        Ok(Some(r)) => TagStep::Foreign(format!(
142            "{tag} already exists on the remote at {r}, not at the merge commit {commit}"
143        )),
144        Ok(None) => match local {
145            Some(l) if l == commit => TagStep::PushExisting,
146            Some(l) => TagStep::Foreign(format!(
147                "{tag} already exists locally at {l}, not at the merge commit {commit}"
148            )),
149            None => TagStep::Create,
150        },
151    }
152}
153
154/// The commit `tag` peels to, out of `git ls-remote --tags <remote>
155/// refs/tags/<tag> refs/tags/<tag>^{}`. The peeled line wins over the tag
156/// object's own.
157pub fn parse_ls_remote(out: &str, tag: &str) -> Option<String> {
158    let direct = format!("refs/tags/{tag}");
159    let peeled = format!("{direct}^{{}}");
160    let mut found = None;
161    for line in out.lines() {
162        let mut it = line.split_whitespace();
163        let (Some(oid), Some(name)) = (it.next(), it.next()) else {
164            continue;
165        };
166        if name == peeled {
167            return Some(oid.to_owned());
168        }
169        if name == direct {
170            found = Some(oid.to_owned());
171        }
172    }
173    found
174}
175
176/// Last `max` bytes of `s`, on a character boundary.
177pub fn tail(s: &str, max: usize) -> String {
178    let s = s.trim_end();
179    if s.len() <= max {
180        return s.to_owned();
181    }
182    let mut cut = s.len() - max;
183    while !s.is_char_boundary(cut) {
184        cut += 1;
185    }
186    format!("...{}", &s[cut..])
187}
188
189/// Where a job keeps its worktree and full command output.
190pub fn job_dir(home: &Path, key: &str) -> PathBuf {
191    home.join("release-local").join(crate::notices::id_of(key))
192}
193
194fn env_of(job: &Job) -> Vec<(&'static str, String)> {
195    vec![
196        ("MAGI_RELEASE_VERSION", job.version.clone()),
197        ("MAGI_RELEASE_TAG", job.tag()),
198        ("MAGI_RELEASE_COMMIT", job.commit.clone()),
199        ("MAGI_RELEASE_PR", job.pr_url.clone()),
200    ]
201}
202
203/// Persists the job; `false` means it could not be, and nothing may run.
204pub type Save<'a> = &'a mut (dyn FnMut(&Job) -> bool + Send);
205
206/// Where and how a job runs.
207pub struct Env<'a> {
208    /// Primary checkout; only used to add and remove the detached worktree.
209    pub repo: &'a Path,
210    /// magi home, for the job's directory.
211    pub home: &'a Path,
212    /// Names the job's directory (the pull request key).
213    pub key: &'a str,
214    /// Remote the tag is pushed to (`[merge] remote`).
215    pub remote: &'a str,
216    /// [`crate::config::Config::shell`].
217    pub shell: &'a [String],
218    /// The `[release]` table.
219    pub release: &'a Release,
220}
221
222/// Drive `job` to the end or to the first failure. `Err` carries the reason it
223/// stopped; the job itself already says how far it got. Never retries.
224pub async fn run_job(env: &Env<'_>, job: &mut Job, save: Save<'_>) -> Result<()> {
225    let dir = job_dir(env.home, env.key);
226    let wt = dir.join("wt");
227    let result = drive(env, &dir, &wt, job, save).await;
228    // A held job keeps its checkout: a resumed command (say `gh release
229    // create`) needs what an earlier one built (`target/release/...`), and a
230    // fresh checkout would skip the build and attach nothing. Only a finished
231    // release cleans up.
232    if job.finished {
233        let _ = git::worktree_remove(env.repo, &wt).await;
234    }
235    result
236}
237
238async fn drive(env: &Env<'_>, dir: &Path, wt: &Path, job: &mut Job, save: Save<'_>) -> Result<()> {
239    let (repo, remote) = (env.repo, env.remote);
240    if job.commit.is_empty() {
241        bail!("the merge commit of {} is unknown", job.pr_url);
242    }
243    // Resuming after progress reuses the checkout the finished steps ran in
244    // (it is no longer "clean": their output is there, which is the point).
245    let progressed = job.tag_done || job.done > 0;
246    let reusable = progressed
247        && wt.exists()
248        && git::rev_parse(wt, "HEAD").await.ok().as_deref() == Some(job.commit.as_str());
249    if !reusable {
250        if wt.exists() {
251            let _ = git::worktree_remove(repo, wt).await;
252            let _ = std::fs::remove_dir_all(wt);
253        }
254        // The merge commit exists on the remote; this checkout may not have it
255        // yet. A failed fetch is only fatal if the commit is still missing.
256        let fetched = git::git_raw(repo, &["fetch", "--quiet", remote]).await?;
257        if !git::rev_exists(repo, &format!("{}^{{commit}}", job.commit)).await {
258            bail!(
259                "the merge commit {} is not available locally (git fetch {remote}: {})",
260                job.commit,
261                fetched.stderr
262            );
263        }
264        git::worktree_add_detached(repo, wt, &job.commit)
265            .await
266            .context("check out the merge commit")?;
267        let head = git::rev_parse(wt, "HEAD").await?;
268        if head != job.commit {
269            bail!(
270                "the release checkout is at {head}, not the merge commit {}",
271                job.commit
272            );
273        }
274        if !git::is_clean(wt).await? {
275            bail!("the release checkout is not clean");
276        }
277    }
278    if let Ok(toml) = std::fs::read_to_string(wt.join("Cargo.toml")) {
279        match crate::bump::current_version(&toml) {
280            Ok(v) if v == job.version => {}
281            Ok(v) => bail!(
282                "Cargo.toml at the merge commit says {v}, but the release is {}; not tagging",
283                job.version
284            ),
285            Err(e) => bail!("cannot read the version at the merge commit: {e:#}"),
286        }
287    }
288
289    if !job.tag_done {
290        job.running = Some("tag".to_owned());
291        if !save(job) {
292            bail!("could not record progress; nothing was run");
293        }
294        let note = tag(wt, remote, job).await?;
295        job.log.push(StepLog {
296            name: "tag".to_owned(),
297            code: Some(0),
298            tail: note,
299        });
300        job.tag_done = true;
301        job.running = None;
302        if !save(job) {
303            bail!("the tag was pushed but progress could not be recorded");
304        }
305    }
306
307    let release = env.release;
308    let timeout = Duration::from_secs(release.timeout_minutes.max(1) * 60);
309    while job.done < release.commands.len() {
310        let n = job.done;
311        let command = &release.commands[n];
312        job.running = Some(format!("command {}", n + 1));
313        if !save(job) {
314            bail!("could not record progress; nothing was run");
315        }
316        let (code, output) = run_command(wt, env.shell, command, job, timeout).await;
317        let _ = std::fs::create_dir_all(dir);
318        let _ = std::fs::write(dir.join(format!("release-{}.out", n + 1)), &output);
319        job.log.push(StepLog {
320            name: command.clone(),
321            code,
322            tail: tail(&output, TAIL),
323        });
324        if code != Some(0) {
325            let why = match code {
326                Some(c) => format!("command {} `{command}` exited {c}", n + 1),
327                None => format!(
328                    "command {} `{command}` did not finish (timeout is {} minute(s))",
329                    n + 1,
330                    release.timeout_minutes
331                ),
332            };
333            // `failed` is saved together with the cleared step: a stop between
334            // the two must never read as an unmarked, retryable job.
335            job.failed = Some(why.clone());
336            job.running = None;
337            let _ = save(job);
338            bail!("{why}");
339        }
340        job.running = None;
341        job.done += 1;
342        if !save(job) {
343            bail!(
344                "command {} succeeded but progress could not be recorded",
345                n + 1
346            );
347        }
348    }
349    job.finished = true;
350    let _ = save(job);
351    Ok(())
352}
353
354/// Read the tag's state and act on [`tag_step`]. Returns a note for the log.
355async fn tag(wt: &Path, remote: &str, job: &Job) -> Result<String> {
356    let name = job.tag();
357    let listed = git::git_raw(
358        wt,
359        &[
360            "ls-remote",
361            "--tags",
362            remote,
363            &format!("refs/tags/{name}"),
364            &format!("refs/tags/{name}^{{}}"),
365        ],
366    )
367    .await?;
368    let remote_tag = if listed.ok() {
369        Ok(parse_ls_remote(&listed.stdout, &name))
370    } else {
371        Err(listed.stderr.clone())
372    };
373    let local = git::git_raw(
374        wt,
375        &[
376            "rev-parse",
377            "-q",
378            "--verify",
379            &format!("refs/tags/{name}^{{commit}}"),
380        ],
381    )
382    .await?;
383    let local = local.ok().then(|| local.stdout.clone());
384    let step = tag_step(
385        &name,
386        &job.commit,
387        remote_tag
388            .as_ref()
389            .map(|o| o.as_deref())
390            .map_err(String::as_str),
391        local.as_deref(),
392    );
393    match step {
394        TagStep::Present => Ok(format!("{name} already on the remote at the merge commit")),
395        TagStep::Foreign(why) | TagStep::Unreadable(why) => bail!("{why}"),
396        TagStep::Create | TagStep::PushExisting => {
397            if step == TagStep::Create {
398                // An annotated tag needs a committer. Use the operator's own
399                // identity; only when git has none (a bare service account) fall
400                // back to a neutral one rather than failing the release.
401                let has_ident = git::git_raw(wt, &["var", "GIT_COMMITTER_IDENT"])
402                    .await?
403                    .ok();
404                let mut args = vec![];
405                if !has_ident {
406                    args.extend(["-c", "user.name=magi", "-c", "user.email=magi@localhost"]);
407                }
408                args.extend(["tag", "-a", name.as_str(), "-m", name.as_str(), &job.commit]);
409                let out = git::git_raw(wt, &args).await?;
410                if !out.ok() {
411                    bail!("git tag {name}: {}", out.stderr);
412                }
413            }
414            // No `+`, no `--force`: an existing remote tag is refused by git.
415            let refspec = format!("refs/tags/{name}:refs/tags/{name}");
416            let out = git::git_raw(wt, &["push", remote, &refspec]).await?;
417            if !out.ok() {
418                bail!("git push {remote} {refspec}: {}", out.stderr);
419            }
420            Ok(format!("pushed {name} at {}", job.commit))
421        }
422    }
423}
424
425/// Run one command in `wt`; `(None, ..)` when it could not run or timed out.
426async fn run_command(
427    wt: &Path,
428    shell: &[String],
429    command: &str,
430    job: &Job,
431    timeout: Duration,
432) -> (Option<i32>, String) {
433    let Some((prog, args)) = shell.split_first() else {
434        return (None, "no shell is configured".to_owned());
435    };
436    let mut cmd = tokio::process::Command::new(prog);
437    cmd.args(args)
438        .arg(command)
439        .current_dir(wt)
440        .kill_on_drop(true)
441        .stdin(std::process::Stdio::null());
442    for (k, v) in env_of(job) {
443        cmd.env(k, v);
444    }
445    cmd.quiet();
446    match tokio::time::timeout(timeout, cmd.output()).await {
447        Err(_) => (None, "timed out".to_owned()),
448        Ok(Err(e)) => (None, format!("could not run: {e}")),
449        Ok(Ok(out)) => {
450            let mut text = String::from_utf8_lossy(&out.stdout).into_owned();
451            text.push_str(&String::from_utf8_lossy(&out.stderr));
452            (out.status.code(), text)
453        }
454    }
455}
456
457#[cfg(test)]
458mod tests {
459    use super::*;
460
461    const C: &str = "aaaa";
462
463    #[test]
464    fn the_tag_decision_never_touches_an_existing_tag() {
465        let t = "v1.0.0";
466        assert_eq!(tag_step(t, C, Ok(None), None), TagStep::Create);
467        assert_eq!(tag_step(t, C, Ok(None), Some(C)), TagStep::PushExisting);
468        assert_eq!(tag_step(t, C, Ok(Some(C)), None), TagStep::Present);
469        assert_eq!(tag_step(t, C, Ok(Some(C)), Some("bbbb")), TagStep::Present);
470        assert!(matches!(
471            tag_step(t, C, Ok(Some("bbbb")), Some(C)),
472            TagStep::Foreign(_)
473        ));
474        assert!(matches!(
475            tag_step(t, C, Ok(None), Some("bbbb")),
476            TagStep::Foreign(_)
477        ));
478        // Unreadable is not absent.
479        assert!(matches!(
480            tag_step(t, C, Err("boom"), None),
481            TagStep::Unreadable(_)
482        ));
483    }
484
485    #[test]
486    fn ls_remote_prefers_the_peeled_commit() {
487        let out = "1111\trefs/tags/v1.0.0\n2222\trefs/tags/v1.0.0^{}\n3333\trefs/tags/v1.0.0-rc\n";
488        assert_eq!(parse_ls_remote(out, "v1.0.0").as_deref(), Some("2222"));
489        assert_eq!(
490            parse_ls_remote("1111\trefs/tags/v1.0.0\n", "v1.0.0").as_deref(),
491            Some("1111")
492        );
493        assert_eq!(parse_ls_remote("", "v1.0.0"), None);
494    }
495
496    #[test]
497    fn version_comes_from_the_release_branch() {
498        assert_eq!(
499            version_from_branch("chore/release-v1.2.3").as_deref(),
500            Some("1.2.3")
501        );
502        assert_eq!(version_from_branch("chore/release-v"), None);
503        assert_eq!(version_from_branch("feat/x"), None);
504    }
505
506    #[test]
507    fn a_started_and_unfinished_step_is_interrupted_not_retried() {
508        let mut j = Job::new("1.0.0", "u", C);
509        assert!(!j.interrupted());
510        j.running = Some("command 1".to_owned());
511        assert!(j.interrupted());
512        j.failed = Some("x".to_owned());
513        assert!(!j.interrupted());
514        j.resume();
515        assert!(!j.interrupted() && j.failed.is_none());
516    }
517
518    #[test]
519    fn tail_keeps_the_end_on_a_char_boundary() {
520        assert_eq!(tail("abc", 10), "abc");
521        assert_eq!(tail("abcdef", 3), "...def");
522        // `é` is two bytes: cutting inside it moves forward to the boundary.
523        assert_eq!(tail("aéb", 2), "...b");
524    }
525
526    /// Run git in `dir`, panicking on failure.
527    fn g(dir: &Path, args: &[&str]) -> String {
528        let out = std::process::Command::new("git")
529            .args(args)
530            .current_dir(dir)
531            .quiet()
532            .env("GIT_AUTHOR_NAME", "t")
533            .env("GIT_AUTHOR_EMAIL", "t@t")
534            .env("GIT_COMMITTER_NAME", "t")
535            .env("GIT_COMMITTER_EMAIL", "t@t")
536            .output()
537            .unwrap();
538        assert!(
539            out.status.success(),
540            "git {args:?}: {}",
541            String::from_utf8_lossy(&out.stderr)
542        );
543        String::from_utf8_lossy(&out.stdout).trim().to_owned()
544    }
545
546    struct Fixture {
547        _dir: tempfile::TempDir,
548        repo: PathBuf,
549        remote: PathBuf,
550        home: PathBuf,
551        commit: String,
552    }
553
554    fn fixture() -> Fixture {
555        let dir = tempfile::tempdir().unwrap();
556        let remote = dir.path().join("remote.git");
557        let repo = dir.path().join("repo");
558        let home = dir.path().join("home");
559        std::fs::create_dir_all(&remote).unwrap();
560        std::fs::create_dir_all(&repo).unwrap();
561        g(&remote, &["init", "--bare", "-q"]);
562        g(&repo, &["init", "-q", "-b", "main"]);
563        std::fs::write(
564            repo.join("Cargo.toml"),
565            "[package]\nname = \"x\"\nversion = \"1.0.0\"\n",
566        )
567        .unwrap();
568        g(&repo, &["add", "."]);
569        g(&repo, &["commit", "-q", "-m", "init"]);
570        g(
571            &repo,
572            &["remote", "add", "origin", remote.to_str().unwrap()],
573        );
574        g(&repo, &["push", "-q", "origin", "main"]);
575        let commit = g(&repo, &["rev-parse", "HEAD"]);
576        Fixture {
577            _dir: dir,
578            repo,
579            remote,
580            home,
581            commit,
582        }
583    }
584
585    fn release(commands: &[&str]) -> Release {
586        Release {
587            mode: crate::config::ReleaseMode::Local,
588            commands: commands.iter().map(|s| (*s).to_owned()).collect(),
589            timeout_minutes: 1,
590        }
591    }
592
593    async fn go(f: &Fixture, rel: &Release, job: &mut Job) -> Result<()> {
594        let shell = vec!["sh".to_owned(), "-c".to_owned()];
595        let env = Env {
596            repo: &f.repo,
597            home: &f.home,
598            key: "o/r#1",
599            remote: "origin",
600            shell: &shell,
601            release: rel,
602        };
603        run_job(&env, job, &mut |_: &Job| true).await
604    }
605
606    #[tokio::test]
607    async fn a_release_tags_once_runs_every_command_and_keeps_the_output() {
608        let f = fixture();
609        let mut job = Job::new("1.0.0", "https://github.com/o/r/pull/1", &f.commit);
610        let rel = release(&["echo tag=$MAGI_RELEASE_TAG", "true"]);
611        go(&f, &rel, &mut job).await.unwrap();
612        assert!(job.finished && job.tag_done);
613        assert_eq!(job.done, 2);
614        assert!(job.log[1].tail.contains("tag=v1.0.0"), "{:?}", job.log);
615        assert_eq!(
616            g(&f.remote, &["rev-parse", "refs/tags/v1.0.0^{commit}"]),
617            f.commit
618        );
619        let out = job_dir(&f.home, "o/r#1").join("release-1.out");
620        assert!(std::fs::read_to_string(out).unwrap().contains("tag=v1.0.0"));
621    }
622
623    #[tokio::test]
624    async fn the_first_failure_stops_the_list_and_a_resume_skips_what_succeeded() {
625        let f = fixture();
626        let mut job = Job::new("1.0.0", "u", &f.commit);
627        let rel = release(&["true", "exit 3", "echo never"]);
628        let err = go(&f, &rel, &mut job).await.unwrap_err().to_string();
629        assert!(err.contains("exited 3"), "{err}");
630        assert_eq!(job.done, 1);
631        assert!(job.tag_done && !job.finished);
632        assert!(!job.log.iter().any(|l| l.tail.contains("never")));
633
634        // The owner fixed it and retried: the tag is not redone, the first
635        // command is not repeated.
636        job.resume();
637        let rel = release(&["true", "true", "echo now"]);
638        let tags_before = g(&f.remote, &["tag", "-l"]);
639        go(&f, &rel, &mut job).await.unwrap();
640        assert!(job.finished);
641        assert_eq!(job.done, 3);
642        assert_eq!(tags_before, g(&f.remote, &["tag", "-l"]));
643        assert_eq!(job.log.iter().filter(|l| l.name == "true").count(), 2);
644    }
645
646    #[tokio::test]
647    async fn a_failure_is_never_saved_as_a_plain_unmarked_job_and_the_checkout_survives() {
648        let f = fixture();
649        let mut job = Job::new("1.0.0", "u", &f.commit);
650        let rel = release(&["touch built.txt", "exit 1"]);
651        let shell = vec!["sh".to_owned(), "-c".to_owned()];
652        let env = Env {
653            repo: &f.repo,
654            home: &f.home,
655            key: "o/r#1",
656            remote: "origin",
657            shell: &shell,
658            release: &rel,
659        };
660        let mut seen: Vec<Job> = Vec::new();
661        run_job(&env, &mut job, &mut |j: &Job| {
662            seen.push(j.clone());
663            true
664        })
665        .await
666        .unwrap_err();
667        // Every saved state either is mid-step or carries the failure.
668        assert!(
669            seen.iter()
670                .all(|j| j.running.is_some() || j.failed.is_some() || j.done > 0 || j.tag_done)
671        );
672        assert!(seen.last().unwrap().failed.is_some());
673        // The first command's output is still there for the resumed one.
674        let wt = job_dir(&f.home, "o/r#1").join("wt");
675        assert!(wt.join("built.txt").exists());
676        job.resume();
677        let rel = release(&["touch built.txt", "test -f built.txt"]);
678        go(&f, &rel, &mut job).await.unwrap();
679        assert!(job.finished);
680        assert!(!wt.exists(), "a finished release removes its checkout");
681    }
682
683    #[tokio::test]
684    async fn an_existing_tag_elsewhere_is_never_pushed_over_and_runs_nothing() {
685        let f = fixture();
686        // A foreign v1.0.0 on the remote, at a different commit.
687        std::fs::write(f.repo.join("x"), "x").unwrap();
688        g(&f.repo, &["add", "."]);
689        g(&f.repo, &["commit", "-q", "-m", "other"]);
690        let other = g(&f.repo, &["rev-parse", "HEAD"]);
691        g(&f.repo, &["tag", "v1.0.0", &other]);
692        g(&f.repo, &["push", "-q", "origin", "refs/tags/v1.0.0"]);
693
694        let mut job = Job::new("1.0.0", "u", &f.commit);
695        let rel = release(&["echo ran > ran.txt"]);
696        let err = go(&f, &rel, &mut job).await.unwrap_err().to_string();
697        assert!(err.contains("already exists"), "{err}");
698        assert!(!job.tag_done && job.done == 0);
699        assert_eq!(
700            g(&f.remote, &["rev-parse", "refs/tags/v1.0.0^{commit}"]),
701            other
702        );
703    }
704
705    #[tokio::test]
706    async fn a_remote_tag_at_the_merge_commit_is_not_pushed_again() {
707        let f = fixture();
708        g(&f.repo, &["tag", "v1.0.0", &f.commit]);
709        g(&f.repo, &["push", "-q", "origin", "refs/tags/v1.0.0"]);
710        let mut job = Job::new("1.0.0", "u", &f.commit);
711        go(&f, &release(&["true"]), &mut job).await.unwrap();
712        assert!(job.finished);
713        assert!(job.log[0].tail.contains("already on the remote"));
714    }
715
716    #[tokio::test]
717    async fn a_version_that_disagrees_with_the_merge_commit_is_not_tagged() {
718        let f = fixture();
719        let mut job = Job::new("2.0.0", "u", &f.commit);
720        let err = go(&f, &release(&[]), &mut job)
721            .await
722            .unwrap_err()
723            .to_string();
724        assert!(err.contains("not tagging"), "{err}");
725        assert_eq!(g(&f.remote, &["tag", "-l"]), "");
726    }
727}