gunnar-server 2.0.0

The gunnar git server: SSH and smart-HTTP git transports and the gRPC control plane. The appliance half, extracted from gunnar-cli so the server code lives in its own crate.
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
//! **The build runner** (roadmap phase 3).
//!
//! A push lands. The runner materialises exactly that commit, builds it with no
//! network, hashes what came out, and signs a [`BuildRecord`] naming the commit
//! it was made from — which is the join that lets `gunnar verify` walk from an
//! artifact to a line of source.
//!
//! ```text
//!   RefUpdated ──▶ checkout(commit) ──▶ sandbox(build) ──▶ artifacts ──▶ signed record ──▶ log
//! ```
//!
//! # The recipe is in the commit, on purpose
//!
//! `gunnar-build.toml` is read **out of the tree being built**, not off the
//! runner's disk. So the recipe is part of what was pushed, part of what was
//! signed, and part of what a verifier can go back and look at. A recipe kept
//! beside the runner would be a build input nobody attested to — the one input
//! that decides what every other input does.
//!
//! # `network = false` is a fact here, not a claim
//!
//! `crate::sandbox` puts the build in a network namespace of its own. The
//! runner **refuses to build** when it cannot, rather than building anyway and
//! recording `network: true`: a server that quietly downgrades is a server whose
//! records mean different things on different machines. `--allow-network` is the
//! explicit opt-out, and it is recorded in the signed record.
//!
//! # What it is not
//!
//! Not a scheduler, not a queue, not a fleet. One build at a time, in the
//! process that watched the event. Scaling it horizontally is the phase-5 item
//! and it is a different problem: this is the part that has to be *right*.

use std::path::{Path, PathBuf};

use anyhow::{Context, Result};
use gunnar_store::git_storage_trait::{GitServe, ObjType};

use crate::stores::ServerStores;

/// The file a repository carries to say how it is built.
pub const RECIPE_FILE: &str = "gunnar-build.toml";

/// How a repository is built.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Recipe {
    /// What this build is called, in the record.
    pub name: String,
    /// The program to run, in the checkout.
    pub command: String,
    pub args: Vec<String>,
    /// Paths, relative to the checkout, that the build produces.
    ///
    /// Empty means **discover**: every ordinary file directly under
    /// `target/release` that is executable. That is the default recipe's shape,
    /// and naming them explicitly is what a repository does when it wants
    /// something else.
    pub artifacts: Vec<String>,
}

impl Recipe {
    /// **The default recipe**, for a cargo workspace that carries none.
    ///
    /// `--locked` above all: a build that may re-resolve is a build whose
    /// inputs are not the ones in the commit, and the SBOM this pairs with is
    /// read from the same lockfile. Without it the two describe different
    /// builds and nothing says so.
    pub fn cargo_default() -> Recipe {
        Recipe {
            name: "cargo build --release --locked".to_owned(),
            command: "cargo".to_owned(),
            args: vec![
                "build".to_owned(),
                "--release".to_owned(),
                "--locked".to_owned(),
            ],
            artifacts: Vec::new(),
        }
    }

    /// Parse `gunnar-build.toml`.
    ///
    /// A **small, total** reader rather than a TOML dependency, for the reason
    /// `gunnar_core::sbom`'s lockfile reader is one: this crate is in the
    /// appliance graph. `key = "value"` and `key = ["a", "b"]`, and anything it
    /// does not understand is skipped rather than guessed at.
    pub fn parse(text: &str) -> Result<Recipe> {
        let mut recipe = Recipe {
            name: String::new(),
            command: String::new(),
            args: Vec::new(),
            artifacts: Vec::new(),
        };
        for line in text.lines() {
            let line = line.trim();
            if line.is_empty() || line.starts_with('#') || line.starts_with('[') {
                continue;
            }
            let Some((key, value)) = line.split_once('=') else {
                continue;
            };
            let value = value.trim();
            let list = |value: &str| -> Vec<String> {
                value
                    .trim_start_matches('[')
                    .trim_end_matches(']')
                    .split(',')
                    .map(|item| item.trim().trim_matches('"').to_owned())
                    .filter(|item| !item.is_empty())
                    .collect()
            };
            match key.trim() {
                "name" => recipe.name = value.trim_matches('"').to_owned(),
                "command" => recipe.command = value.trim_matches('"').to_owned(),
                "args" => recipe.args = list(value),
                "artifacts" => recipe.artifacts = list(value),
                _ => {}
            }
        }
        anyhow::ensure!(
            !recipe.command.is_empty(),
            "{RECIPE_FILE} names no `command`, so there is nothing to run. A recipe that builds \
             nothing is not a smaller recipe — it is a build that reports success having done \
             nothing."
        );
        if recipe.name.is_empty() {
            recipe.name = recipe.command.clone();
        }
        Ok(recipe)
    }
}

/// What one build did.
#[derive(Debug, Clone)]
pub struct Built {
    pub record: gunnar_core::BuildRecord,
    /// Where the checkout and its output are, for a caller that wants to look.
    pub workdir: PathBuf,
}

/// Materialise `tree` into `into`, from the store and nothing else.
///
/// No `git checkout`, no worktree, no gix: the tree is read through `GitServe`,
/// which is the one contract every engine has. That is also why the runner can
/// build a repository the archive engine stores, which has no working copy on
/// disk to check out from.
fn checkout(
    git: &dyn GitServe,
    hash: gix_hash::Kind,
    tree: gix_hash::ObjectId,
    into: &Path,
) -> Result<usize> {
    std::fs::create_dir_all(into).with_context(|| format!("creating {}", into.display()))?;
    let (kind, bytes) = git
        .read(tree.as_bytes())?
        .with_context(|| format!("tree {tree} is not in this repository"))?;
    anyhow::ensure!(kind == ObjType::Tree, "{tree} is not a tree");

    let mut files = 0usize;
    for entry in gunnar_store::object::TreeIter::from_bytes(&bytes, hash).named() {
        let entry = entry.with_context(|| format!("decoding tree {tree}"))?;
        let name = String::from_utf8_lossy(entry.name).into_owned();
        // The sandbox boundary starts here: a tree entry named `..` or with a
        // separator in it would write outside the checkout, and a repository is
        // something anybody with push access controls.
        anyhow::ensure!(
            name != "." && name != ".." && !name.contains('/'),
            "tree {tree} carries an entry named {name:?}, which would write outside the checkout"
        );
        let path = into.join(&name);
        match entry.kind {
            gunnar_store::object::EntryKind::Tree => {
                files += checkout(git, hash, entry.oid, &path)?;
            }
            gunnar_store::object::EntryKind::Commit => {
                // A gitlink names a commit in another repository. There is
                // nothing here to write, and writing an empty directory would
                // make the build look like it had a submodule checked out.
                continue;
            }
            kind => {
                let Some((ObjType::Blob, blob)) = git.read(entry.oid.as_bytes())? else {
                    anyhow::bail!("{} is not a blob in this repository", entry.oid);
                };
                if kind == gunnar_store::object::EntryKind::Link {
                    // A symlink's blob is its target. Recreated as a symlink so
                    // a build that follows one behaves as it did for the author.
                    let target = String::from_utf8_lossy(&blob).into_owned();
                    let _ = std::fs::remove_file(&path);
                    std::os::unix::fs::symlink(&target, &path)
                        .with_context(|| format!("linking {}", path.display()))?;
                } else {
                    std::fs::write(&path, &blob)
                        .with_context(|| format!("writing {}", path.display()))?;
                    if kind == gunnar_store::object::EntryKind::BlobExecutable {
                        use std::os::unix::fs::PermissionsExt as _;
                        std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o755))
                            .with_context(|| format!("chmod {}", path.display()))?;
                    }
                }
                files += 1;
            }
        }
    }
    Ok(files)
}

/// The recipe a checkout carries, or the default one it qualifies for.
fn recipe_for(checkout: &Path) -> Result<Recipe> {
    let path = checkout.join(RECIPE_FILE);
    if path.exists() {
        let text = std::fs::read_to_string(&path)
            .with_context(|| format!("reading {}", path.display()))?;
        return Recipe::parse(&text)
            .with_context(|| format!("{} is not a usable recipe", path.display()));
    }
    anyhow::ensure!(
        checkout.join("Cargo.toml").exists(),
        "this repository carries no {RECIPE_FILE} and is not a cargo workspace, so there is no \
         recipe and no default that fits. Add a {RECIPE_FILE} naming a `command`."
    );
    Ok(Recipe::cargo_default())
}

/// Every artifact the recipe produced.
fn collect(checkout: &Path, recipe: &Recipe) -> Result<Vec<PathBuf>> {
    if !recipe.artifacts.is_empty() {
        let mut out = Vec::new();
        for named in &recipe.artifacts {
            let path = checkout.join(named);
            anyhow::ensure!(
                path.is_file(),
                "the recipe names {named} and the build did not produce it. A build that \
                 reports success without its own declared output is a build nobody checked."
            );
            out.push(path);
        }
        return Ok(out);
    }
    // Discovery, for the default recipe: executables directly under
    // `target/release`. Not recursive — `target/release/deps` and `build/` are
    // cargo's own scratch, and shipping them as artifacts would attest to
    // hundreds of files nobody asked for.
    use std::os::unix::fs::PermissionsExt as _;
    let dir = checkout.join("target").join("release");
    let mut out = Vec::new();
    if let Ok(entries) = std::fs::read_dir(&dir) {
        for entry in entries.flatten() {
            let path = entry.path();
            let Ok(meta) = entry.metadata() else { continue };
            if meta.is_file() && meta.permissions().mode() & 0o111 != 0 {
                out.push(path);
            }
        }
    }
    out.sort();
    anyhow::ensure!(
        !out.is_empty(),
        "the build produced no executable in {}. Either it built nothing, or its output is \
         somewhere the default recipe does not look — name the artifacts in {RECIPE_FILE}.",
        dir.display()
    );
    Ok(out)
}

/// **Build one commit and attest to what came out.**
///
/// `allow_network` is the explicit opt-out from hermeticity, and it goes into
/// the signed record. Without it a runner that cannot enter the sandbox
/// **fails** rather than building: a record that says `network: false` must mean
/// it on every machine, or it means nothing on any of them.
pub fn build_commit(
    stores: &ServerStores,
    repo: &gunnar_core::RepoRef,
    commit: gix_hash::ObjectId,
    workspace: &Path,
    builder: &str,
    allow_network: bool,
) -> Result<Built> {
    let entry = stores
        .catalog()
        .get(repo)?
        .with_context(|| format!("no such repository: {repo}"))?;
    let store = stores.repo_store(&entry)?;
    let git = store.serve();
    let hash = store.hash_kind();

    let (kind, bytes) = git
        .read(commit.as_bytes())?
        .with_context(|| format!("commit {commit} is not in {repo}"))?;
    anyhow::ensure!(kind == ObjType::Commit, "{commit} is not a commit");
    let tree = gunnar_store::object::commit(&bytes, hash)
        .with_context(|| format!("decoding commit {commit}"))?
        .tree;

    let workdir = workspace
        .join(repo.namespace())
        .join(repo.name())
        .join(commit.to_string());
    // A fresh checkout every time. Reusing one would make the build depend on
    // what the *last* build left behind, which is the first thing that makes a
    // "reproducible" build not one.
    let _ = std::fs::remove_dir_all(&workdir);
    let files = checkout(git, hash, tree, &workdir)?;
    anyhow::ensure!(
        files > 0,
        "commit {commit} has an empty tree; there is nothing to build"
    );

    let recipe = recipe_for(&workdir)?;

    if !allow_network {
        if let Err(err) = crate::sandbox::available() {
            anyhow::bail!(
                "this build was to be hermetic and the sandbox cannot be entered: {}\n\n\
                 Refusing rather than building with the network up. A record that says \
                 `network: false` has to mean it on every machine. Pass --allow-network to \
                 build anyway; the record will say so.",
                err.explain()
            );
        }
    }

    let started = gunnar_grpc::bus::now_unix_ms();
    let mut command = hardened_command(&recipe.command, &workdir, allow_network);
    command
        .args(&recipe.args)
        .env("CARGO_NET_OFFLINE", "true")
        .env("GUNNAR_BUILD_COMMIT", commit.to_string())
        .env("GUNNAR_BUILD_REPO", repo.to_string());
    let output = command
        .output()
        .with_context(|| format!("running {}", recipe.command))?;
    let finished = gunnar_grpc::bus::now_unix_ms();

    if let Some(err) = output
        .status
        .code()
        .and_then(crate::sandbox::error_from_exit)
    {
        anyhow::bail!("the build never started: {}", err.explain());
    }
    anyhow::ensure!(
        output.status.success(),
        "the build failed ({}):\n{}",
        output.status,
        String::from_utf8_lossy(&output.stderr)
    );

    let artifacts = collect(&workdir, &recipe)?;
    let subjects = artifacts
        .iter()
        .map(|path| -> Result<gunnar_core::Subject> {
            Ok(gunnar_core::Subject {
                name: path
                    .strip_prefix(&workdir)
                    .unwrap_or(path)
                    .display()
                    .to_string(),
                sha256: gunnar_core::provenance::digest_file(path)
                    .with_context(|| format!("hashing {}", path.display()))?,
            })
        })
        .collect::<Result<Vec<_>>>()?;

    // The lockfile is a **material**: it is what decided every dependency, and
    // it is in the commit, so naming it here is what lets the SBOM beside this
    // record be checked against the same bytes.
    let mut materials = Vec::new();
    let lock = workdir.join("Cargo.lock");
    if lock.is_file() {
        materials.push(gunnar_core::Subject {
            name: "Cargo.lock".to_owned(),
            sha256: gunnar_core::provenance::digest_file(&lock)?,
        });
    }

    let record = gunnar_core::BuildRecord {
        builder: builder.to_owned(),
        source_repo: repo.to_string(),
        source_commit: commit.to_string(),
        recipe: recipe.name.clone(),
        toolchain: toolchain(&recipe.command, &workdir, allow_network),
        network: allow_network,
        materials,
        subjects,
        started_at_unix_ms: started,
        finished_at_unix_ms: finished,
    };

    let key = stores.provenance_key()?;
    let signed = gunnar_core::provenance::sign_build(&key, record.clone())
        .context("signing the build record")?;
    let mut line = serde_json::to_string(&signed)?;
    line.push('\n');
    let _framed = gunnar_core::txlog::sink::append_line(stores.build_log().records_path(), line.trim_end_matches('\n'))?;
    use std::io::Write as _;
    let mut file = std::fs::OpenOptions::new()
        .create(true)
        .append(true)
        .open(stores.build_log().records_path())?;
    file.write_all(line.as_bytes())?;

    Ok(Built { record, workdir })
}

/// **The one constructor of every process a build runs.**
///
/// GUN-006. `build_commit` built its `Command` with `env_clear`,
/// `current_dir(&workdir)` and `pre_exec(enter_sandbox)`, and then
/// [`toolchain`] ran the SAME attacker-named program a second time with
/// `Command::new(command).arg("--version")` — no cleared environment, no
/// working directory, no sandbox. One hardened exec and one bare one, of a
/// `recipe.command` read out of the pushed tree and validated against nothing
/// but `!is_empty()`.
///
/// The escape was worth more than the build: the bare child ran as the server's
/// uid with the network up, so it could read `$GUNNAR_SECRET_KEK` or
/// `secrets.kek` from disk and decrypt the sealed store — the SSH host key and
/// the provenance signing key — and then sign `BuildRecord`s of its own.
///
/// So the hardening is a function both call sites must go through, and
/// `the_runner_has_exactly_one_process_constructor` holds a third from being
/// added beside them. What is NOT here is the build's own environment
/// (`CARGO_NET_OFFLINE`, `GUNNAR_BUILD_*`): those are facts about a build and
/// `toolchain` is not one, and adding them here would make the version probe
/// claim to be a build step.
fn hardened_command(program: &str, workdir: &Path, allow_network: bool) -> std::process::Command {
    // A32-14, a process by design: the build runner runs a repository's own declared build program, hardened (cleared environment, no network unless allowed); it is the feature, and no cargo or sh of Gunnar's own.
    let mut command = std::process::Command::new(program);
    command
        .current_dir(workdir)
        // A clean-ish environment: the build's inputs should be the commit, not
        // the operator's shell. `PATH` and `HOME` stay because a toolchain
        // needs to be findable and cargo needs somewhere to put its caches.
        .env_clear()
        .env("PATH", std::env::var_os("PATH").unwrap_or_default())
        .env("HOME", std::env::var_os("HOME").unwrap_or_default());
    if !allow_network {
        use std::os::unix::process::CommandExt as _;
        // The hook is bound OUTSIDE the `unsafe` block on purpose: an `unsafe`
        // block extends lexically into a closure written inside it, so nesting
        // the two made the inner one redundant (`unused_unsafe`) and its SAFETY
        // note unenforced. Bound here, each `unsafe` covers exactly one call.
        let enter_sandbox = || {
            // SAFETY: `pre_exec` runs in the forked child, single-threaded,
            // before `exec`.
            if let Err(err) = unsafe { crate::sandbox::enter() } {
                crate::sandbox::leave_child(crate::sandbox::exit_code(err));
            }
            Ok(())
        };
        // SAFETY: `sandbox::enter` allocates nothing and takes no lock, which
        // is the whole requirement `pre_exec` imposes.
        unsafe {
            command.pre_exec(enter_sandbox);
        }
    }
    command
}

/// What the toolchain says it is, or an honest blank.
///
/// Asked of the tool that ran, not read from a config: a `rust-toolchain.toml`
/// says what was *requested* and this says what answered. Empty when the tool
/// will not say, because a made-up version in a signed record is worse than no
/// version at all.
///
/// Through [`hardened_command`], because this runs the pushed tree's own
/// `recipe.command` — see GUN-006 — and its stdout goes verbatim into the
/// SIGNED `toolchain` field of the build record.
fn toolchain(command: &str, workdir: &Path, allow_network: bool) -> String {
    hardened_command(command, workdir, allow_network)
        .arg("--version")
        .output()
        .ok()
        .filter(|out| out.status.success())
        .map(|out| String::from_utf8_lossy(&out.stdout).trim().to_owned())
        .unwrap_or_default()
}

#[cfg(test)]
mod tests {
    use super::*;

    /// ★ **A watch that ends is said** (A28 M18 follow-up: serve.rs dropped the result with
    /// `let _ =`). An error is an ALERT at ERROR with its cause; any other end is a WARN that
    /// says how many pushes were built and that no more are.
    #[test]
    fn a_watch_that_ends_is_said_and_an_error_is_an_alert() {
        let (level, words) = said_end(&Err(anyhow::anyhow!("building a runtime for the runner's subscription: no threads")));
        assert_eq!(level, tracing::Level::ERROR);
        assert!(words.starts_with("ALERT runner"), "{words}");
        assert!(words.contains("no threads") && words.contains("no longer built"), "{words}");
        let (level, words) = said_end(&Ok(3));
        assert_eq!(level, tracing::Level::WARN);
        assert!(words.contains("3 build") && words.contains("no longer built"), "{words}");
    }

    /// ★ The watch's own end goes through [`said_end`]: a watch bounded to no builds ends at
    /// once, and what the server says of it is the WARN, not nothing.
    #[test]
    fn the_watch_returns_its_end_for_the_server_to_say() {
        let dir = tempfile::tempdir().expect("a temp dir");
        let stores = ServerStores::open(dir.path()).expect("stores");
        let bus = gunnar_grpc::EventBus::new(4);
        let config = WatchConfig { workspace: dir.path().join("ws"), builder: "runner".into(), allow_network: true, refs: Vec::new(), max_builds: Some(0) };
        let end = watch(&stores, &bus, &config, |_| {});
        assert_eq!(said_end(&end).0, tracing::Level::WARN, "{:?}", said_end(&end));
    }

    #[test]
    fn a_recipe_parses_and_one_with_no_command_is_refused() {
        let recipe = Recipe::parse(
            r#"
            name = "the appliance"
            command = "make"
            args = ["-j", "4"]
            artifacts = ["out/gunnar"]
            "#,
        )
        .expect("a usable recipe");
        assert_eq!(recipe.name, "the appliance");
        assert_eq!(recipe.command, "make");
        assert_eq!(recipe.args, vec!["-j", "4"]);
        assert_eq!(recipe.artifacts, vec!["out/gunnar"]);

        // A recipe that builds nothing must be refused rather than treated as a
        // smaller one: it would report success having done nothing.
        let err = Recipe::parse("name = \"empty\"").expect_err("no command");
        assert!(format!("{err}").contains("command"), "{err}");
    }

    #[test]
    fn a_recipe_with_no_name_is_named_after_its_command() {
        let recipe = Recipe::parse("command = \"cargo\"").expect("usable");
        assert_eq!(recipe.name, "cargo");
    }

    /// **GUN-006: every process a build runs goes through one constructor.**
    ///
    /// The finding was not that `toolchain` was written carelessly — it was
    /// that there were TWO places that exec a `recipe.command` read out of the
    /// pushed tree, and only one of them was hardened. A fix that hardened the
    /// second one would leave a third free to forget again, which is why the
    /// hardening became a function and why this counts.
    ///
    /// Source-level on purpose. The property is "there is one constructor",
    /// and no behaviour of a single call can assert that; the file is the
    /// evidence. `include_str!` reads THIS file at compile time, so the guard
    /// cannot drift from what it guards.
    ///
    /// Mutation-proven 2026-09-11: put `Command::new` back in `toolchain` and
    /// the count is 2.
    #[test]
    fn the_runner_has_exactly_one_process_constructor() {
        let source = include_str!("runner.rs");
        let constructors = source
            .lines()
            // A doc comment quoting the old line is not a call site, and
            // neither is a string literal — including the needle on this very
            // line, which is why the quote character is spelled rather than
            // written.
            .filter(|line| !line.trim_start().starts_with("//"))
            .filter(|line| !line.contains(char::from(34)))
            .filter(|line| line.contains("std::process::Command::new("))
            .count();
        assert_eq!(
            constructors, 1,
            "every exec a build performs must go through `hardened_command`; \
             found {constructors} fully-qualified process constructors in runner.rs"
        );
        // …and the spelling cannot be dodged by importing the type, which
        // would make the count above read 1 while a second exec sat beside it.
        // Spelled in two halves so the needle is not in the file it searches.
        // A guard that trips on its own source proves nothing about the code
        // it guards.
        assert!(
            !source.contains(concat!("use std::process::", "Command;")),
            "runner.rs imports the process builder by name; the count above \
             only sees the fully-qualified spelling"
        );
    }

    /// The hardening is applied, and it is applied to a process that can be
    /// watched: a cleared environment and the working directory the build was
    /// checked out into.
    ///
    /// `/usr/bin/env` with no arguments prints the environment it was given,
    /// and `/bin/pwd` prints the directory it was started in. Both are
    /// coreutils and both are already assumed by the suites that drive `git`
    /// and `ssh`; neither needs a shell.
    #[test]
    fn the_hardened_command_clears_the_environment_and_sets_the_directory() {
        let dir = tempfile::tempdir().expect("a temp dir");
        // A marker the caller's environment carries and the child must not.
        // Set on the CHILD-to-be's parent — this process — through the same
        // `Command` API, so no global state is mutated under cargo's test
        // threads.
        let seen = |program: &str| -> Option<String> {
            let out = hardened_command(program, dir.path(), true)
                .env("GUNNAR_TEST_LEAK", "this must not cross")
                .output()
                .ok()?;
            out.status
                .success()
                .then(|| String::from_utf8_lossy(&out.stdout).into_owned())
        };
        // `.env()` after `env_clear()` puts the marker back deliberately: it
        // proves the child is readable at all, so an empty answer below cannot
        // be mistaken for a cleared environment.
        let Some(environment) = seen("/usr/bin/env") else {
            return; // no coreutils on this box; nothing to measure
        };
        assert!(
            environment.contains("GUNNAR_TEST_LEAK=this must not cross"),
            "the fixture cannot see the child's environment at all: {environment:?}"
        );
        // …and nothing else of the caller's. `PATH` and `HOME` are the two
        // `hardened_command` puts back by name.
        for line in environment.lines().filter(|l| !l.is_empty()) {
            let name = line.split('=').next().unwrap_or_default();
            assert!(
                matches!(name, "PATH" | "HOME" | "GUNNAR_TEST_LEAK"),
                "the build child inherited {name} from the operator's shell"
            );
        }

        if let Some(cwd) = seen("/bin/pwd") {
            assert_eq!(
                std::fs::canonicalize(cwd.trim()).ok(),
                std::fs::canonicalize(dir.path()).ok(),
                "the build child did not start in the checkout"
            );
        }
    }

    #[test]
    fn the_default_recipe_locks_its_dependencies() {
        // `--locked` is the whole point of the default: without it the build
        // may re-resolve, and the SBOM read from the lockfile then describes a
        // different build with nothing saying so.
        assert!(
            Recipe::cargo_default()
                .args
                .contains(&"--locked".to_owned()),
            "the default cargo recipe does not pass --locked"
        );
    }
}

// ───────────────────────────────────────────────────────────────────────────
// Watching
// ───────────────────────────────────────────────────────────────────────────

/// How the runner is configured when it watches rather than being told.
#[derive(Debug, Clone)]
pub struct WatchConfig {
    /// Where checkouts and build output go.
    pub workspace: PathBuf,
    /// The ACL principal every record is attributed to.
    pub builder: String,
    /// Build with the network up. Off by default; recorded when on.
    pub allow_network: bool,
    /// Only build these refs. Empty means every branch.
    ///
    /// A list and not a pattern: a pattern language here would be a second
    /// ref-matching implementation beside `gunnar-policy`'s, and two of those
    /// can disagree about which branch is protected.
    pub refs: Vec<String>,
    /// Stop after this many builds. `None` watches for ever; a bound is what a
    /// test uses, and what an operator uses to drain one queue and stop.
    pub max_builds: Option<usize>,
}

/// What one watched build turned into.
#[derive(Debug)]
pub enum Watched {
    /// A build ran and attested.
    Built(Box<Built>),
    /// A ref moved and the runner did not build it.
    Skipped {
        repo: String,
        r#ref: String,
        why: String,
    },
    /// A build was attempted and failed. **Not fatal to the watch**: one
    /// repository's broken build must not stop every other repository's.
    Failed {
        repo: String,
        commit: String,
        why: String,
    },
}

/// Should this ref update be built?
fn wanted(config: &WatchConfig, ref_name: &str) -> bool {
    // Branches only. A tag move is not a new state of the code — the commit it
    // points at was already pushed and already built — and a runner that built
    // both would attest twice to one commit under two recipes' worth of noise.
    if !ref_name.starts_with("refs/heads/") {
        return false;
    }
    config.refs.is_empty() || config.refs.iter().any(|want| want == ref_name)
}

/// **How the watch ended, as the server says it** (A28 M18 follow-up): the server runs
/// [`watch`] for its whole life, so ANY end while the server runs means pushes are no longer
/// built. Never silent: an error is an `ALERT runner` at ERROR, any other end a WARN.
pub fn said_end(end: &Result<usize>) -> (tracing::Level, String) {
    match end {
        Err(err) => (tracing::Level::ERROR, format!("ALERT runner: the build runner stopped watching this server's pushes: {err:#}; pushes are no longer built until the server restarts")),
        Ok(built) => (tracing::Level::WARN, format!("the build runner stopped watching this server's pushes after {built} build(s); pushes are no longer built until the server restarts")),
    }
}

/// **Watch the event bus and build what lands.**
///
/// Blocking, and deliberately so: the runner is one build at a time in the
/// process that watched the event. A queue and a pool are the phase-5 problem;
/// this is the part that has to be right, and "right" is easier to see when
/// there is one of it.
///
/// Every outcome — built, skipped, failed — is handed to `report` rather than
/// logged and dropped, so a caller can assert on what happened. A build that
/// fails does **not** stop the watch: one repository's broken recipe must not
/// take every other repository's builds with it.
pub fn watch(
    stores: &ServerStores,
    bus: &gunnar_grpc::EventBus,
    config: &WatchConfig,
    mut report: impl FnMut(Watched),
) -> Result<usize> {
    // `tokio_stream`, not `futures_util`: the stream is a `BroadcastStream` and
    // tokio-stream is already resolved here. One stream crate, not two.
    use tokio_stream::StreamExt as _;

    let filter = gunnar_grpc::EventFilter {
        namespaces: Vec::new(),
        repos: Vec::new(),
        kinds: vec![gunnar_proto::EventKind::RefUpdated],
        include_attestation_object_ids: false,
    };
    let mut stream = bus.subscribe(filter);
    let runtime = tokio::runtime::Builder::new_current_thread()
        .enable_all()
        .build()
        .context("building a runtime for the runner's subscription")?;

    let mut built = 0usize;
    while config.max_builds.is_none_or(|max| built < max) {
        let Some(event) = runtime.block_on(stream.next()) else {
            break;
        };
        // The stream yields a `Result` because a gRPC subscriber's does. An
        // error here is the subscription itself ending, not one event going
        // wrong, so it stops the watch rather than being skipped past.
        let Ok(event) = event else { break };
        let Some(gunnar_proto::event::Payload::RefUpdated(update)) = &event.payload else {
            continue;
        };
        let Some(repo) = update.repo.as_ref() else {
            continue;
        };
        let key = format!("{}/{}", repo.namespace, repo.repo);
        if !wanted(config, &update.ref_name) {
            report(Watched::Skipped {
                repo: key,
                r#ref: update.ref_name.clone(),
                why: "not a branch this runner builds".to_owned(),
            });
            continue;
        }
        let Ok(commit) = gix_hash::ObjectId::from_hex(update.new_oid.as_bytes()) else {
            report(Watched::Skipped {
                repo: key,
                r#ref: update.ref_name.clone(),
                why: format!("{} is not an object id", update.new_oid),
            });
            continue;
        };
        let Ok(repo_ref) = gunnar_core::RepoRef::new(&repo.namespace, &repo.repo) else {
            continue;
        };
        match build_commit(
            stores,
            &repo_ref,
            commit,
            &config.workspace,
            &config.builder,
            config.allow_network,
        ) {
            Ok(outcome) => {
                built += 1;
                report(Watched::Built(Box::new(outcome)));
            }
            Err(err) => report(Watched::Failed {
                repo: key,
                commit: commit.to_string(),
                why: format!("{err:#}"),
            }),
        }
    }
    Ok(built)
}