Skip to main content

magi/
release_local.rs

1//! Releasing without GitHub Actions (`[release] mode = "local"`).
2//!
3//! On a repository whose Actions never run, `auto-tag.yml` and `release.yml`
4//! never fire, so nothing tags or publishes after the release pull request
5//! merges. In local mode [`crate::release_watch`] merges that pull request on
6//! the owner's approval and then drives the [`Job`] defined here, from a clean
7//! detached checkout of the merge commit:
8//!
9//! 1. create and push the `vX.Y.Z` tag - **never over a tag that exists**
10//!    ([`tag_step`] is the whole decision);
11//! 2. run `[release] commands` one at a time, stopping at the first failure.
12//!
13//! - **Progress is saved before every step**, through the caller's `save`. A
14//!   step that was started and never finished (a crash, a kill) is
15//!   [`Job::interrupted`]: magi cannot know whether `gh release create` got as
16//!   far as the forge, so it holds for the owner instead of repeating it.
17//! - **A failure is a hold, not a retry.** [`run_job`] returns the reason; the
18//!   watcher records it, raises a notice and asks. Resuming skips the tag and
19//!   every command that already succeeded.
20//! - **Environment, not templates.** Commands see `MAGI_RELEASE_VERSION`,
21//!   `MAGI_RELEASE_TAG`, `MAGI_RELEASE_COMMIT` and `MAGI_RELEASE_PR`; teravars
22//!   renders the whole config before any release exists, so `{{ version }}`
23//!   cannot work.
24//! - Only `git` and the commands the operator wrote run; there is no API path.
25
26use std::path::{Path, PathBuf};
27use std::time::Duration;
28
29use anyhow::{Context, Result, bail};
30use serde::{Deserialize, Serialize};
31
32use crate::config::Release;
33use crate::git;
34use crate::proc::Quiet as _;
35
36/// Bytes of a command's output kept in the record; the whole of it goes to
37/// `release-<n>.out` beside the job.
38const TAIL: usize = 4_000;
39
40/// Head of a release branch name, before the version.
41pub const BRANCH_PREFIX: &str = "chore/release-v";
42
43/// What happened in one step, kept in the record.
44#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
45#[serde(default)]
46pub struct StepLog {
47    /// `tag` or the command line.
48    pub name: String,
49    /// Exit code; `None` for a signal or a timeout.
50    pub code: Option<i32>,
51    /// Last bytes of the combined output.
52    pub tail: String,
53    /// Absolute path of the file holding the whole output, when one was kept.
54    pub output: Option<String>,
55}
56
57/// One release of one merged pull request, resumable.
58#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
59#[serde(default)]
60pub struct Job {
61    /// `X.Y.Z`, without the `v`.
62    pub version: String,
63    /// The merged release pull request.
64    pub pr_url: String,
65    /// The commit the pull request merged as; what the tag points at.
66    pub commit: String,
67    /// The tag exists on the remote at `commit`.
68    pub tag_done: bool,
69    /// Commands that finished successfully.
70    pub done: usize,
71    /// The step started and not yet finished (`tag` or `command N`).
72    pub running: Option<String>,
73    /// Why the job stopped. Held until the owner decides.
74    pub failed: Option<String>,
75    /// Everything ran.
76    pub finished: bool,
77    /// What each step that ran said, in order.
78    pub log: Vec<StepLog>,
79}
80
81impl Job {
82    /// A job that has not started.
83    pub fn new(version: &str, pr_url: &str, commit: &str) -> Self {
84        Self {
85            version: version.to_owned(),
86            pr_url: pr_url.to_owned(),
87            commit: commit.to_owned(),
88            ..Self::default()
89        }
90    }
91
92    /// The tag name, `vX.Y.Z`.
93    pub fn tag(&self) -> String {
94        format!("v{}", self.version)
95    }
96
97    /// Stopped in the middle of a step with nothing recorded about how it
98    /// ended: the outcome is unknown, so it is not retried on its own.
99    pub fn interrupted(&self) -> bool {
100        self.running.is_some() && self.failed.is_none() && !self.finished
101    }
102
103    /// Owner chose to retry: forget the stop and resume from the progress.
104    pub fn resume(&mut self) {
105        self.failed = None;
106        self.running = None;
107    }
108}
109
110/// `1.2.3` out of `chore/release-v1.2.3`.
111pub fn version_from_branch(branch: &str) -> Option<String> {
112    let v = branch.strip_prefix(BRANCH_PREFIX)?;
113    (!v.is_empty()).then(|| v.to_owned())
114}
115
116/// What to do about the tag. Pure.
117#[derive(Debug, Clone, PartialEq, Eq)]
118pub enum TagStep {
119    /// Neither the remote nor this checkout has it: create it and push.
120    Create,
121    /// A local tag already at the commit but not on the remote: push it.
122    PushExisting,
123    /// The remote already has it at the commit: nothing to do.
124    Present,
125    /// A tag of that name points elsewhere. Never touched.
126    Foreign(String),
127    /// The remote could not be read, so "does not exist" is not known.
128    Unreadable(String),
129}
130
131/// Decide the tag step from what was read. `remote` is the commit the remote's
132/// tag peels to (`Ok(None)`: absent), or the reason it could not be read;
133/// `local` is the commit of a local tag of that name.
134pub fn tag_step(
135    tag: &str,
136    commit: &str,
137    remote: std::result::Result<Option<&str>, &str>,
138    local: Option<&str>,
139) -> TagStep {
140    match remote {
141        Err(e) => TagStep::Unreadable(format!("could not read the remote's {tag}: {e}")),
142        Ok(Some(r)) if r == commit => TagStep::Present,
143        Ok(Some(r)) => TagStep::Foreign(format!(
144            "{tag} already exists on the remote at {r}, not at the merge commit {commit}"
145        )),
146        Ok(None) => match local {
147            Some(l) if l == commit => TagStep::PushExisting,
148            Some(l) => TagStep::Foreign(format!(
149                "{tag} already exists locally at {l}, not at the merge commit {commit}"
150            )),
151            None => TagStep::Create,
152        },
153    }
154}
155
156/// The commit `tag` peels to, out of `git ls-remote --tags <remote>
157/// refs/tags/<tag> refs/tags/<tag>^{}`. The peeled line wins over the tag
158/// object's own.
159pub fn parse_ls_remote(out: &str, tag: &str) -> Option<String> {
160    let direct = format!("refs/tags/{tag}");
161    let peeled = format!("{direct}^{{}}");
162    let mut found = None;
163    for line in out.lines() {
164        let mut it = line.split_whitespace();
165        let (Some(oid), Some(name)) = (it.next(), it.next()) else {
166            continue;
167        };
168        if name == peeled {
169            return Some(oid.to_owned());
170        }
171        if name == direct {
172            found = Some(oid.to_owned());
173        }
174    }
175    found
176}
177
178/// Last `max` bytes of `s`, on a character boundary.
179pub fn tail(s: &str, max: usize) -> String {
180    let s = s.trim_end();
181    if s.len() <= max {
182        return s.to_owned();
183    }
184    let mut cut = s.len() - max;
185    while !s.is_char_boundary(cut) {
186        cut += 1;
187    }
188    format!("...{}", &s[cut..])
189}
190
191/// Where a job keeps its worktree and full command output.
192pub fn job_dir(home: &Path, key: &str) -> PathBuf {
193    home.join("release-local").join(crate::notices::id_of(key))
194}
195
196fn env_of(job: &Job) -> Vec<(&'static str, String)> {
197    vec![
198        ("MAGI_RELEASE_VERSION", job.version.clone()),
199        ("MAGI_RELEASE_TAG", job.tag()),
200        ("MAGI_RELEASE_COMMIT", job.commit.clone()),
201        ("MAGI_RELEASE_PR", job.pr_url.clone()),
202    ]
203}
204
205/// Persists the job; `false` means it could not be, and nothing may run.
206pub type Save<'a> = &'a mut (dyn FnMut(&Job) -> bool + Send);
207
208/// Where and how a job runs.
209pub struct Env<'a> {
210    /// Primary checkout; only used to add and remove the detached worktree.
211    pub repo: &'a Path,
212    /// magi home, for the job's directory.
213    pub home: &'a Path,
214    /// Names the job's directory (the pull request key).
215    pub key: &'a str,
216    /// Remote the tag is pushed to (`[merge] remote`).
217    pub remote: &'a str,
218    /// [`crate::config::Config::shell`].
219    pub shell: &'a [String],
220    /// The `[release]` table.
221    pub release: &'a Release,
222}
223
224/// Drive `job` to the end or to the first failure. `Err` carries the reason it
225/// stopped; the job itself already says how far it got. Never retries.
226pub async fn run_job(env: &Env<'_>, job: &mut Job, save: Save<'_>) -> Result<()> {
227    let dir = job_dir(env.home, env.key);
228    let wt = dir.join("wt");
229    let result = drive(env, &dir, &wt, job, save).await;
230    // A held job keeps its checkout: a resumed command (say `gh release
231    // create`) needs what an earlier one built (`target/release/...`), and a
232    // fresh checkout would skip the build and attach nothing. Only a finished
233    // release cleans up.
234    if job.finished {
235        let _ = git::worktree_remove(env.repo, &wt).await;
236    }
237    result
238}
239
240async fn drive(env: &Env<'_>, dir: &Path, wt: &Path, job: &mut Job, save: Save<'_>) -> Result<()> {
241    let (repo, remote) = (env.repo, env.remote);
242    if job.commit.is_empty() {
243        bail!("the merge commit of {} is unknown", job.pr_url);
244    }
245    // Resuming after progress reuses the checkout the finished steps ran in
246    // (it is no longer "clean": their output is there, which is the point).
247    let progressed = job.tag_done || job.done > 0;
248    let reusable = progressed
249        && wt.exists()
250        && git::rev_parse(wt, "HEAD").await.ok().as_deref() == Some(job.commit.as_str());
251    if !reusable {
252        if wt.exists() {
253            let _ = git::worktree_remove(repo, wt).await;
254            let _ = std::fs::remove_dir_all(wt);
255        }
256        // The merge commit exists on the remote; this checkout may not have it
257        // yet. A failed fetch is only fatal if the commit is still missing.
258        let fetched = git::git_raw(repo, &["fetch", "--quiet", remote]).await?;
259        if !git::rev_exists(repo, &format!("{}^{{commit}}", job.commit)).await {
260            bail!(
261                "the merge commit {} is not available locally (git fetch {remote}: {})",
262                job.commit,
263                fetched.stderr
264            );
265        }
266        git::worktree_add_detached(repo, wt, &job.commit)
267            .await
268            .context("check out the merge commit")?;
269        let head = git::rev_parse(wt, "HEAD").await?;
270        if head != job.commit {
271            bail!(
272                "the release checkout is at {head}, not the merge commit {}",
273                job.commit
274            );
275        }
276        if !git::is_clean(wt).await? {
277            bail!("the release checkout is not clean");
278        }
279    }
280    if let Ok(toml) = std::fs::read_to_string(wt.join("Cargo.toml")) {
281        match crate::bump::current_version(&toml) {
282            Ok(v) if v == job.version => {}
283            Ok(v) => bail!(
284                "Cargo.toml at the merge commit says {v}, but the release is {}; not tagging",
285                job.version
286            ),
287            Err(e) => bail!("cannot read the version at the merge commit: {e:#}"),
288        }
289    }
290
291    if !job.tag_done {
292        job.running = Some("tag".to_owned());
293        if !save(job) {
294            bail!("could not record progress; nothing was run");
295        }
296        let note = tag(wt, remote, job).await?;
297        job.log.push(StepLog {
298            name: "tag".to_owned(),
299            code: Some(0),
300            tail: note,
301            output: None,
302        });
303        job.tag_done = true;
304        job.running = None;
305        if !save(job) {
306            bail!("the tag was pushed but progress could not be recorded");
307        }
308    }
309
310    let release = env.release;
311    let timeout = Duration::from_secs(release.timeout_minutes.max(1) * 60);
312    while job.done < release.commands.len() {
313        let n = job.done;
314        let command = &release.commands[n];
315        job.running = Some(format!("command {}", n + 1));
316        if !save(job) {
317            bail!("could not record progress; nothing was run");
318        }
319        let (code, output) = run_command(wt, env.shell, command, job, timeout).await;
320        let _ = std::fs::create_dir_all(dir);
321        let out_path = dir.join(format!("release-{}.out", n + 1));
322        let kept = std::fs::write(&out_path, &output).is_ok();
323        job.log.push(StepLog {
324            name: command.clone(),
325            code,
326            tail: tail(&output, TAIL),
327            output: kept.then(|| out_path.display().to_string()),
328        });
329        if code != Some(0) {
330            let why = match code {
331                Some(c) => format!("command {} `{command}` exited {c}", n + 1),
332                None => format!(
333                    "command {} `{command}` did not finish (timeout is {} minute(s))",
334                    n + 1,
335                    release.timeout_minutes
336                ),
337            };
338            // `failed` is saved together with the cleared step: a stop between
339            // the two must never read as an unmarked, retryable job.
340            job.failed = Some(why.clone());
341            job.running = None;
342            let _ = save(job);
343            bail!("{why}");
344        }
345        job.running = None;
346        job.done += 1;
347        if !save(job) {
348            bail!(
349                "command {} succeeded but progress could not be recorded",
350                n + 1
351            );
352        }
353    }
354    job.finished = true;
355    let _ = save(job);
356    Ok(())
357}
358
359/// Read the tag's state and act on [`tag_step`]. Returns a note for the log.
360async fn tag(wt: &Path, remote: &str, job: &Job) -> Result<String> {
361    let name = job.tag();
362    let listed = git::git_raw(
363        wt,
364        &[
365            "ls-remote",
366            "--tags",
367            remote,
368            &format!("refs/tags/{name}"),
369            &format!("refs/tags/{name}^{{}}"),
370        ],
371    )
372    .await?;
373    let remote_tag = if listed.ok() {
374        Ok(parse_ls_remote(&listed.stdout, &name))
375    } else {
376        Err(listed.stderr.clone())
377    };
378    let local = git::git_raw(
379        wt,
380        &[
381            "rev-parse",
382            "-q",
383            "--verify",
384            &format!("refs/tags/{name}^{{commit}}"),
385        ],
386    )
387    .await?;
388    let local = local.ok().then(|| local.stdout.clone());
389    let step = tag_step(
390        &name,
391        &job.commit,
392        remote_tag
393            .as_ref()
394            .map(|o| o.as_deref())
395            .map_err(String::as_str),
396        local.as_deref(),
397    );
398    match step {
399        TagStep::Present => Ok(format!("{name} already on the remote at the merge commit")),
400        TagStep::Foreign(why) | TagStep::Unreadable(why) => bail!("{why}"),
401        TagStep::Create | TagStep::PushExisting => {
402            if step == TagStep::Create {
403                // An annotated tag needs a committer. Use the operator's own
404                // identity; only when git has none (a bare service account) fall
405                // back to a neutral one rather than failing the release.
406                let has_ident = git::git_raw(wt, &["var", "GIT_COMMITTER_IDENT"])
407                    .await?
408                    .ok();
409                let mut args = vec![];
410                if !has_ident {
411                    args.extend(["-c", "user.name=magi", "-c", "user.email=magi@localhost"]);
412                }
413                args.extend(["tag", "-a", name.as_str(), "-m", name.as_str(), &job.commit]);
414                let out = git::git_raw(wt, &args).await?;
415                if !out.ok() {
416                    bail!("git tag {name}: {}", out.stderr);
417                }
418            }
419            // No `+`, no `--force`: an existing remote tag is refused by git.
420            let refspec = format!("refs/tags/{name}:refs/tags/{name}");
421            let out = git::git_raw(wt, &["push", remote, &refspec]).await?;
422            if !out.ok() {
423                bail!("git push {remote} {refspec}: {}", out.stderr);
424            }
425            Ok(format!("pushed {name} at {}", job.commit))
426        }
427    }
428}
429
430/// Run one command in `wt`; `(None, ..)` when it could not run or timed out.
431async fn run_command(
432    wt: &Path,
433    shell: &[String],
434    command: &str,
435    job: &Job,
436    timeout: Duration,
437) -> (Option<i32>, String) {
438    let Some((prog, args)) = shell.split_first() else {
439        return (None, "no shell is configured".to_owned());
440    };
441    let mut cmd = tokio::process::Command::new(prog);
442    cmd.args(args)
443        .arg(command)
444        .current_dir(wt)
445        .kill_on_drop(true)
446        .stdin(std::process::Stdio::null());
447    for (k, v) in env_of(job) {
448        cmd.env(k, v);
449    }
450    cmd.quiet();
451    match tokio::time::timeout(timeout, cmd.output()).await {
452        Err(_) => (None, "timed out".to_owned()),
453        Ok(Err(e)) => (None, format!("could not run: {e}")),
454        Ok(Ok(out)) => {
455            let mut text = String::from_utf8_lossy(&out.stdout).into_owned();
456            text.push_str(&String::from_utf8_lossy(&out.stderr));
457            (out.status.code(), text)
458        }
459    }
460}
461
462#[cfg(test)]
463mod tests {
464    use super::*;
465
466    const C: &str = "aaaa";
467
468    #[test]
469    fn the_tag_decision_never_touches_an_existing_tag() {
470        let t = "v1.0.0";
471        assert_eq!(tag_step(t, C, Ok(None), None), TagStep::Create);
472        assert_eq!(tag_step(t, C, Ok(None), Some(C)), TagStep::PushExisting);
473        assert_eq!(tag_step(t, C, Ok(Some(C)), None), TagStep::Present);
474        assert_eq!(tag_step(t, C, Ok(Some(C)), Some("bbbb")), TagStep::Present);
475        assert!(matches!(
476            tag_step(t, C, Ok(Some("bbbb")), Some(C)),
477            TagStep::Foreign(_)
478        ));
479        assert!(matches!(
480            tag_step(t, C, Ok(None), Some("bbbb")),
481            TagStep::Foreign(_)
482        ));
483        // Unreadable is not absent.
484        assert!(matches!(
485            tag_step(t, C, Err("boom"), None),
486            TagStep::Unreadable(_)
487        ));
488    }
489
490    #[test]
491    fn ls_remote_prefers_the_peeled_commit() {
492        let out = "1111\trefs/tags/v1.0.0\n2222\trefs/tags/v1.0.0^{}\n3333\trefs/tags/v1.0.0-rc\n";
493        assert_eq!(parse_ls_remote(out, "v1.0.0").as_deref(), Some("2222"));
494        assert_eq!(
495            parse_ls_remote("1111\trefs/tags/v1.0.0\n", "v1.0.0").as_deref(),
496            Some("1111")
497        );
498        assert_eq!(parse_ls_remote("", "v1.0.0"), None);
499    }
500
501    #[test]
502    fn version_comes_from_the_release_branch() {
503        assert_eq!(
504            version_from_branch("chore/release-v1.2.3").as_deref(),
505            Some("1.2.3")
506        );
507        assert_eq!(version_from_branch("chore/release-v"), None);
508        assert_eq!(version_from_branch("feat/x"), None);
509    }
510
511    #[test]
512    fn a_started_and_unfinished_step_is_interrupted_not_retried() {
513        let mut j = Job::new("1.0.0", "u", C);
514        assert!(!j.interrupted());
515        j.running = Some("command 1".to_owned());
516        assert!(j.interrupted());
517        j.failed = Some("x".to_owned());
518        assert!(!j.interrupted());
519        j.resume();
520        assert!(!j.interrupted() && j.failed.is_none());
521    }
522
523    #[test]
524    fn tail_keeps_the_end_on_a_char_boundary() {
525        assert_eq!(tail("abc", 10), "abc");
526        assert_eq!(tail("abcdef", 3), "...def");
527        // `é` is two bytes: cutting inside it moves forward to the boundary.
528        assert_eq!(tail("aéb", 2), "...b");
529    }
530
531    /// Run git in `dir`, panicking on failure.
532    fn g(dir: &Path, args: &[&str]) -> String {
533        let out = std::process::Command::new("git")
534            .args(args)
535            .current_dir(dir)
536            .quiet()
537            .env("GIT_AUTHOR_NAME", "t")
538            .env("GIT_AUTHOR_EMAIL", "t@t")
539            .env("GIT_COMMITTER_NAME", "t")
540            .env("GIT_COMMITTER_EMAIL", "t@t")
541            .output()
542            .unwrap();
543        assert!(
544            out.status.success(),
545            "git {args:?}: {}",
546            String::from_utf8_lossy(&out.stderr)
547        );
548        String::from_utf8_lossy(&out.stdout).trim().to_owned()
549    }
550
551    struct Fixture {
552        _dir: tempfile::TempDir,
553        repo: PathBuf,
554        remote: PathBuf,
555        home: PathBuf,
556        commit: String,
557    }
558
559    fn fixture() -> Fixture {
560        let dir = tempfile::tempdir().unwrap();
561        let remote = dir.path().join("remote.git");
562        let repo = dir.path().join("repo");
563        let home = dir.path().join("home");
564        std::fs::create_dir_all(&remote).unwrap();
565        std::fs::create_dir_all(&repo).unwrap();
566        g(&remote, &["init", "--bare", "-q"]);
567        g(&repo, &["init", "-q", "-b", "main"]);
568        std::fs::write(
569            repo.join("Cargo.toml"),
570            "[package]\nname = \"x\"\nversion = \"1.0.0\"\n",
571        )
572        .unwrap();
573        g(&repo, &["add", "."]);
574        g(&repo, &["commit", "-q", "-m", "init"]);
575        g(
576            &repo,
577            &["remote", "add", "origin", remote.to_str().unwrap()],
578        );
579        g(&repo, &["push", "-q", "origin", "main"]);
580        let commit = g(&repo, &["rev-parse", "HEAD"]);
581        Fixture {
582            _dir: dir,
583            repo,
584            remote,
585            home,
586            commit,
587        }
588    }
589
590    fn release(commands: &[&str]) -> Release {
591        Release {
592            mode: crate::config::ReleaseMode::Local,
593            commands: commands.iter().map(|s| (*s).to_owned()).collect(),
594            timeout_minutes: 1,
595        }
596    }
597
598    async fn go(f: &Fixture, rel: &Release, job: &mut Job) -> Result<()> {
599        let shell = vec!["sh".to_owned(), "-c".to_owned()];
600        let env = Env {
601            repo: &f.repo,
602            home: &f.home,
603            key: "o/r#1",
604            remote: "origin",
605            shell: &shell,
606            release: rel,
607        };
608        run_job(&env, job, &mut |_: &Job| true).await
609    }
610
611    #[tokio::test]
612    async fn a_release_tags_once_runs_every_command_and_keeps_the_output() {
613        let f = fixture();
614        let mut job = Job::new("1.0.0", "https://github.com/o/r/pull/1", &f.commit);
615        let rel = release(&["echo tag=$MAGI_RELEASE_TAG", "true"]);
616        go(&f, &rel, &mut job).await.unwrap();
617        assert!(job.finished && job.tag_done);
618        assert_eq!(job.done, 2);
619        assert!(job.log[1].tail.contains("tag=v1.0.0"), "{:?}", job.log);
620        assert_eq!(
621            g(&f.remote, &["rev-parse", "refs/tags/v1.0.0^{commit}"]),
622            f.commit
623        );
624        let out = job_dir(&f.home, "o/r#1").join("release-1.out");
625        assert!(std::fs::read_to_string(out).unwrap().contains("tag=v1.0.0"));
626    }
627
628    #[tokio::test]
629    async fn the_first_failure_stops_the_list_and_a_resume_skips_what_succeeded() {
630        let f = fixture();
631        let mut job = Job::new("1.0.0", "u", &f.commit);
632        let rel = release(&["true", "exit 3", "echo never"]);
633        let err = go(&f, &rel, &mut job).await.unwrap_err().to_string();
634        assert!(err.contains("exited 3"), "{err}");
635        assert_eq!(job.done, 1);
636        assert!(job.tag_done && !job.finished);
637        assert!(!job.log.iter().any(|l| l.tail.contains("never")));
638
639        // The owner fixed it and retried: the tag is not redone, the first
640        // command is not repeated.
641        job.resume();
642        let rel = release(&["true", "true", "echo now"]);
643        let tags_before = g(&f.remote, &["tag", "-l"]);
644        go(&f, &rel, &mut job).await.unwrap();
645        assert!(job.finished);
646        assert_eq!(job.done, 3);
647        assert_eq!(tags_before, g(&f.remote, &["tag", "-l"]));
648        assert_eq!(job.log.iter().filter(|l| l.name == "true").count(), 2);
649    }
650
651    #[tokio::test]
652    async fn a_failure_is_never_saved_as_a_plain_unmarked_job_and_the_checkout_survives() {
653        let f = fixture();
654        let mut job = Job::new("1.0.0", "u", &f.commit);
655        let rel = release(&["touch built.txt", "exit 1"]);
656        let shell = vec!["sh".to_owned(), "-c".to_owned()];
657        let env = Env {
658            repo: &f.repo,
659            home: &f.home,
660            key: "o/r#1",
661            remote: "origin",
662            shell: &shell,
663            release: &rel,
664        };
665        let mut seen: Vec<Job> = Vec::new();
666        run_job(&env, &mut job, &mut |j: &Job| {
667            seen.push(j.clone());
668            true
669        })
670        .await
671        .unwrap_err();
672        // Every saved state either is mid-step or carries the failure.
673        assert!(
674            seen.iter()
675                .all(|j| j.running.is_some() || j.failed.is_some() || j.done > 0 || j.tag_done)
676        );
677        assert!(seen.last().unwrap().failed.is_some());
678        // The first command's output is still there for the resumed one.
679        let wt = job_dir(&f.home, "o/r#1").join("wt");
680        assert!(wt.join("built.txt").exists());
681        job.resume();
682        let rel = release(&["touch built.txt", "test -f built.txt"]);
683        go(&f, &rel, &mut job).await.unwrap();
684        assert!(job.finished);
685        assert!(!wt.exists(), "a finished release removes its checkout");
686    }
687
688    #[tokio::test]
689    async fn an_existing_tag_elsewhere_is_never_pushed_over_and_runs_nothing() {
690        let f = fixture();
691        // A foreign v1.0.0 on the remote, at a different commit.
692        std::fs::write(f.repo.join("x"), "x").unwrap();
693        g(&f.repo, &["add", "."]);
694        g(&f.repo, &["commit", "-q", "-m", "other"]);
695        let other = g(&f.repo, &["rev-parse", "HEAD"]);
696        g(&f.repo, &["tag", "v1.0.0", &other]);
697        g(&f.repo, &["push", "-q", "origin", "refs/tags/v1.0.0"]);
698
699        let mut job = Job::new("1.0.0", "u", &f.commit);
700        let rel = release(&["echo ran > ran.txt"]);
701        let err = go(&f, &rel, &mut job).await.unwrap_err().to_string();
702        assert!(err.contains("already exists"), "{err}");
703        assert!(!job.tag_done && job.done == 0);
704        assert_eq!(
705            g(&f.remote, &["rev-parse", "refs/tags/v1.0.0^{commit}"]),
706            other
707        );
708    }
709
710    #[tokio::test]
711    async fn a_remote_tag_at_the_merge_commit_is_not_pushed_again() {
712        let f = fixture();
713        g(&f.repo, &["tag", "v1.0.0", &f.commit]);
714        g(&f.repo, &["push", "-q", "origin", "refs/tags/v1.0.0"]);
715        let mut job = Job::new("1.0.0", "u", &f.commit);
716        go(&f, &release(&["true"]), &mut job).await.unwrap();
717        assert!(job.finished);
718        assert!(job.log[0].tail.contains("already on the remote"));
719    }
720
721    #[tokio::test]
722    async fn a_version_that_disagrees_with_the_merge_commit_is_not_tagged() {
723        let f = fixture();
724        let mut job = Job::new("2.0.0", "u", &f.commit);
725        let err = go(&f, &release(&[]), &mut job)
726            .await
727            .unwrap_err()
728            .to_string();
729        assert!(err.contains("not tagging"), "{err}");
730        assert_eq!(g(&f.remote, &["tag", "-l"]), "");
731    }
732}