1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
#cloud-config
# yah-cloud mirror bootstrap (R092-F2 — containerd + yubaba + cloudflared + tailscaled).
#
# Substitutions (written as {{ KEY }} with inner spaces so these docs survive rendering):
# {{ MACHINE_NAME }} — machine name from .yah/cloud/machines/<name>.toml
# {{ YAH_YUBABA_URL }} — HTTPS URL of the yah-yubaba release tar.gz
# (published by .github/workflows/release.yml per musl target;
# R406-T13: now also contains the kamaji binary +
# yubaba.service + kamaji.service + yubaba.slice)
# {{ YAH_YUBABA_SHA256 }} — lowercase hex sha256 of the tar.gz at YAH_YUBABA_URL
# {{ YUBABA_CHANNEL }} — release channel: stable | beta
# {{ CONTAINERD_VERSION }} — containerd apt version pin (e.g. 1.7.2~3-0~debian-bookworm);
# use DEFAULT_CONTAINERD_VERSION constant for Phase 1 baseline
# {{ HEADSCALE_PREAUTH_KEY }} — Tailscale/Headscale pre-auth key (consumed once at join)
# {{ MESH_LOGIN_SERVER_ARG }} — expands to ` --login-server <url>` when a custom Headscale
# coordinator is configured; empty string for Tailscale SaaS
# {{ TAGS }} — comma-joined machine.mesh_tags for Tailscale advertise-tags
# {{ CLOUDFLARED_BLOCK }} — cloudflared apt-repo install + `cloudflared service install
# <tok>` (token is a positional arg, not a --token flag —
# R330-B29) + `systemctl enable --now cloudflared` when
# machine.cloudflared is set; empty string otherwise
# {{ OPERATOR_BRIDGE_BLOCK }} — tailscaled install + `tailscale up` + ufw allow on tailscale0
# when machine.hosts_operator_bridge = true; empty otherwise
# {{ COSIGN_VERIFY_BLOCK }} — cosign install + `cosign verify-blob` of the yubaba tarball
# against a pinned GitHub-OIDC identity-regexp when
# yubaba_cosign_identity_regexp is set (R330-F19/F20); empty
# string otherwise (sha256 verify stays as the trust gate)
# {{ REGION_ENV_BLOCK }} — writes YUBABA_REGION=<machine.region> into the same
# yubaba.service.d drop-in as YUBABA_CHANNEL (R937-F4) when
# machine.region is set; empty otherwise. yubaba's `--region`
# flag carries a clap `env =` fallback for exactly this — the
# unit's ExecStart ships verbatim across every camp, so a
# per-node region can only reach the daemon via env, never a
# baked-in flag.
#
# The yubaba tarball is curl-fetched at boot (not base64-embedded) because Hetzner caps
# user_data at 32 KiB (R040-F11). The runcmd verifies sha256, extracts, and installs
# /usr/local/bin/yubaba + /usr/local/bin/kamaji + three systemd units under
# /etc/systemd/system/ before the systemd hand-off.
#
# Supervision model (W154, R406-T13): yubaba.service and kamaji.service are sibling
# units, both pinned to yubaba.slice (created by the slice unit, owned by kamaji via
# Slice= directive). Kamaji starts first (UDS server up before yubaba's first Hello);
# yubaba joins on After=kamaji.service. systemd's role is reduced to supervising the
# two siblings — it does NOT see individual workloads, which kamaji owns directly via
# pidfds + cgroup-v2 syscalls under yubaba.slice.
#
# Containerd is the container runtime yubaba drives via gRPC. Podman/podman-compose
# were removed in R092-F2 (yubaba is now the workload lifecycle owner; see yah-yubaba-
# integration-testing.md for the ContainerRuntime abstraction).
#
# Tailscale (operator-bridge) is optional: only installed when machine.hosts_operator_bridge
# is true. The yubaba cluster mesh uses its own WireGuard plane (yah-cluster-mesh.md) and
# does not depend on Tailscale for cluster-internal traffic.
hostname:
preserve_hostname: false
package_update: true
package_upgrade: true
packages:
- wireguard-tools
- chrony
- curl
- ca-certificates
- openssh-server
- ufw
runcmd:
# Disk bounds FIRST, before anything on this box starts writing. yah doctrine:
# no process ever grows on disk without an explicit ceiling — a full disk takes
# the node down no matter how good the software above it is. Debian's journald
# default is 10% of the filesystem (capped at 4G), which is a fraction, not a
# bound. RuntimeMaxUse also bounds RAM, since /run is tmpfs.
- mkdir -p /etc/systemd/journald.conf.d
- sh -c 'printf "[Journal]\nSystemMaxUse=500M\nSystemKeepFree=1G\nSystemMaxFileSize=50M\nRuntimeMaxUse=64M\n" > /etc/systemd/journald.conf.d/10-yah-disk-bounds.conf'
- systemctl restart systemd-journald
# containerd — kamaji's container backend. Installed unpinned: yubaba drives
# it over a stable gRPC API, and an exact apt version pin matches neither the
# Debian repo's suffixed versions (e.g. 1.6.20~ds1) nor newer point releases,
# so it broke first-boot on Debian 12/13 alike (R330-T9).
- apt-get install -y containerd
- systemctl enable --now containerd
# Yubaba + kamaji bundle — fetch + verify + install (R406-T13).
# The tarball contains: yubaba, kamaji, yubaba.service, kamaji.service,
# yubaba.slice. cloud-init lays the binaries under /usr/local/bin and the units
# under /etc/systemd/system/ before systemctl daemon-reload.
#
# R858-F17: turso-backup-hydrate and turso-backup-tail ship in the SAME
# tarball rather than a separate download — they are yah's own co-developed
# oss/turso-backup, not a third-party release like litestream/headscale
# below, so they version-lockstep with kamaji the same way kamaji itself
# does with yubaba. scripts/publish-yubaba-release.sh builds + packages +
# version-gates them the same way it already does kamaji (-p turso-backup,
# oss/turso-backup, one invocation per --bin since both live in one
# package), so every tarball this template fetches carries them from here
# on. SEQUENCING HAZARD, still live: the moment ANY workload spec declares
# a `yah.durability.tier`, kamaji REFUSES to deploy it on a node whose
# kamaji has neither helper configured (kamaji-bin/src/hydrate.rs:170-178,
# src/tail.rs). A release carrying these binaries + the env vars below MUST
# reach every candidate node BEFORE the release that adds the declaration
# to headscale's WorkloadSpec, or the first roll of the second refuses to
# deploy the coordinator. The cp/chmod steps below stay present-checked and
# non-fatal regardless — an OLDER tarball (cut before this landed) must
# still provision cleanly, just without the helpers.
- curl -fsSL -o /tmp/yah-yubaba.tar.gz {{YAH_YUBABA_URL}}
# cosign verify-blob — primary trust gate when yubaba_cosign_identity_regexp is
# set (R330-F19/F20). sha256 line below stays as a redundant integrity check.
- sh -c 'echo "{{YAH_YUBABA_SHA256}} /tmp/yah-yubaba.tar.gz" | sha256sum -c -'
- tar -xzf /tmp/yah-yubaba.tar.gz -C /tmp
- sh -c 'cp /tmp/yubaba-*/yubaba /usr/local/bin/yubaba'
- sh -c 'cp /tmp/yubaba-*/kamaji /usr/local/bin/kamaji'
# Double-quoted, not bare: these two carry a `: ` inside their echo, and a
# colon-space in a BARE YAML scalar makes the whole runcmd entry parse as a
# mapping — cloud-init then skips it silently and the helpers never land.
# `rendered_runcmd_entries_are_all_strings` (cloud_init.rs) is the gate;
# it went red when these were added.
- "sh -c '[ -f /tmp/yubaba-*/turso-backup-hydrate ] && cp /tmp/yubaba-*/turso-backup-hydrate /usr/local/bin/turso-backup-hydrate || echo \"turso-backup-hydrate: absent from this tarball, durability-declaring workloads will refuse to deploy (R858-F17)\" >&2'"
- "sh -c '[ -f /tmp/yubaba-*/turso-backup-tail ] && cp /tmp/yubaba-*/turso-backup-tail /usr/local/bin/turso-backup-tail || echo \"turso-backup-tail: absent from this tarball, durability-declaring workloads will refuse to deploy (R858-F17)\" >&2'"
- chmod +x /usr/local/bin/yubaba /usr/local/bin/kamaji
- sh -c '[ -f /usr/local/bin/turso-backup-hydrate ] && chmod +x /usr/local/bin/turso-backup-hydrate; [ -f /usr/local/bin/turso-backup-tail ] && chmod +x /usr/local/bin/turso-backup-tail; true'
# KAMAJI_HYDRATE_HELPER / KAMAJI_TAIL_HELPER, as Environment= in a drop-in,
# never as ExecStart= flags — see kamaji.service's own warning: a drop-in
# that redeclares ExecStart= silently drops every flag added after it
# (measured on us-south-001, half of the 2026-09-03 outage). Both are
# no-ops until a workload actually declares a durability tier, so this is
# safe to lay down on every node before headscale's declaration ships.
- mkdir -p /etc/systemd/system/kamaji.service.d
- sh -c '[ -f /usr/local/bin/turso-backup-hydrate ] && [ -f /usr/local/bin/turso-backup-tail ] && printf "[Service]\nEnvironment=KAMAJI_HYDRATE_HELPER=/usr/local/bin/turso-backup-hydrate\nEnvironment=KAMAJI_TAIL_HELPER=/usr/local/bin/turso-backup-tail\n" > /etc/systemd/system/kamaji.service.d/50-durability-helpers.conf || true'
- sh -c 'cp /tmp/yubaba-*/yubaba.slice /etc/systemd/system/yubaba.slice'
- sh -c 'cp /tmp/yubaba-*/kamaji.service /etc/systemd/system/kamaji.service'
- sh -c 'cp /tmp/yubaba-*/yubaba.service /etc/systemd/system/yubaba.service'
# litestream-headscale.service — headscale DB replication for whichever node
# holds the ingress owner role. Laid down here and never enabled: leader.rs
# starts it on gaining that role and stops it on losing it. It MUST be
# pre-staged for the same reason headscale.service is (see the coordinator
# pre-stage block below) — yubaba runs ProtectSystem=strict and cannot write
# /etc/systemd/system, so a unit it "installs" at runtime is a silent EROFS.
- sh -c 'cp /tmp/yubaba-*/litestream-headscale.service /etc/systemd/system/litestream-headscale.service'
- chmod 0644 /etc/systemd/system/yubaba.slice /etc/systemd/system/kamaji.service /etc/systemd/system/yubaba.service /etc/systemd/system/litestream-headscale.service
- sh -c 'rm -rf /tmp/yah-yubaba.tar.gz /tmp/yubaba-*'
# litestream — the replicate/restore binary litestream-headscale.service execs
# and leader.rs shells for `litestream restore` before starting the appliance.
# Absent from every node in the fleet as of 2026-09-04, which made the whole
# DB-continuity path inert regardless of configuration.
#
# PINNED TO 0.3.13, NOT LATEST (0.5.x): litestream 0.5 is a rewrite onto the
# LTX format with a changed config schema, and `litestream::generate_config`
# emits the 0.3 `dbs[].replicas[].url` shape. Moving to 0.5 is a real
# migration — new config emitter, and a replica written by 0.3 is not a
# replica 0.5 reads — not a version bump. Checksums below were computed by
# downloading these exact assets, not copied from the release page.
# A BLOCK SCALAR, not a plain multi-line one: the `echo "litestream: ..."`
# below contains a colon-space, which YAML reads as a mapping indicator and
# which broke this template's parse when it was first written inline.
- |-
sh -c '
case "$(dpkg --print-architecture)" in
amd64) LS_ARCH=amd64; LS_SHA=eb75a3de5cab03875cdae9f5f539e6aedadd66607003d9b1e7a9077948818ba0 ;;
arm64) LS_ARCH=arm64; LS_SHA=9585f5a508516bd66af2b2376bab4de256a5ef8e2b73ec760559e679628f2d59 ;;
*) echo "litestream: unsupported arch, skipping" >&2; exit 0 ;;
esac
curl -fsSL -o /tmp/litestream.tar.gz \
"https://github.com/benbjohnson/litestream/releases/download/v0.3.13/litestream-v0.3.13-linux-${LS_ARCH}.tar.gz"
echo "${LS_SHA} /tmp/litestream.tar.gz" | sha256sum -c -
tar -xzf /tmp/litestream.tar.gz -C /usr/local/bin litestream
chmod +x /usr/local/bin/litestream
rm -f /tmp/litestream.tar.gz
'
# headscale — the coordination server's own binary, on EVERY node rather than
# only the one that was promoted (R858-T4).
#
# Why every node: `headscale_appliance::appliance_spec` is a NATIVE workload,
# so kamaji forks `/var/lib/yah-cloud/headscale/headscale` and pulls nothing.
# Until now the binary arrived only via `POST /headscale/deploy` — the promote
# path — which is why us-west-001 had it (51,593,368 bytes, mtime Jun 22) and
# us-south-001 had no binary, no config.yaml and no DB. On 2026-09-03
# ownership moved to south anyway and the appliance ran nowhere for 37 hours.
# A moveable native appliance needs its executable to be a PROVISIONED
# PREREQUISITE on every candidate, not a side effect of having once been the
# coordinator. Staged and never started: leader.rs decides who runs it.
#
# Pinned to 0.23.0 — the same pin `cloud::mesh::HEADSCALE_VERSION` and
# `yubaba::DEFAULT_HEADSCALE_VERSION` already carry, kept in lockstep by
# convention like the config renderers (there is no dependency edge from this
# template to either constant, so a bump has to touch all three).
#
# Checksums were computed by DOWNLOADING these exact assets, not copied off
# the release page — the same standard the litestream block above records. The
# amd64 size matches the binary already on us-west-001 byte-for-byte, which is
# the corroboration that this is the asset the fleet is already running.
#
# A BLOCK SCALAR for the same reason as litestream's: the `echo "headscale:
# ..."` below contains a colon-space, which YAML reads as a mapping indicator.
- |-
sh -c '
case "$(dpkg --print-architecture)" in
amd64) HS_ARCH=amd64; HS_SHA=d9193dad4b070b9b3f6d54c8f14366952944b6e917672c0bc1dfd8f5491287a7 ;;
arm64) HS_ARCH=arm64; HS_SHA=99fa9b2944c50759882b578e78aa11968d6fdec9bbfeced88237a1138b89e9fe ;;
*) echo "headscale: unsupported arch, skipping" >&2; exit 0 ;;
esac
mkdir -p /var/lib/yah-cloud/headscale
curl -fsSL -o /tmp/headscale \
"https://github.com/juanfont/headscale/releases/download/v0.23.0/headscale_0.23.0_linux_${HS_ARCH}"
echo "${HS_SHA} /tmp/headscale" | sha256sum -c -
mv /tmp/headscale /var/lib/yah-cloud/headscale/headscale
chmod +x /var/lib/yah-cloud/headscale/headscale
'
# Channel goes into a drop-in (yubaba.service's ExecStart bakes the default args;
# the channel is operator-tunable per node).
- mkdir -p /etc/systemd/system/yubaba.service.d
- sh -c 'printf "[Service]\nEnvironment=YUBABA_CHANNEL={{YUBABA_CHANNEL}}\n" > /etc/systemd/system/yubaba.service.d/channel.conf'
# Cloudflare Tunnel — public ingress (when machine.cloudflared is set)
# Operator-bridge — Tailscale mesh access (when machine.hosts_operator_bridge = true)
# Firewall — allow SSH; yubaba RPC (7443) reachability depends on mesh role:
# mesh-only (deny public) when joining a mesh, or public on a standalone
# coordinator so the operator can attach + `yah mesh bootstrap` it before any
# mesh exists to reach it over (R330-F28 #13).
- ufw --force enable
- ufw allow 22/tcp
# Coordinator pre-stage (standalone only) — cloud-init runs unsandboxed at boot,
# so it lays down the headscale.service unit + opens ufw 80/443 here. yubaba runs
# under ProtectSystem=strict and CANNOT write /etc/systemd/system or /etc/ufw, so
# `yah mesh bootstrap` only writes config + `systemctl enable --now headscale`
# against this pre-staged unit (R330-F28 #15). Empty for joining nodes.
# /var/lib/yah/yubaba (the secret store, in yubaba.service's ReadWritePaths)
# is created by the unit itself via StateDirectory=yah/yubaba (R589-T2), so no
# pre-mkdir crutch is needed here — the unit no longer fails 226/NAMESPACE.
# Bring up the supervision tree (W154 ordering): slice → kamaji → yubaba.
# kamaji.service has After=yubaba.slice + PartOf=yubaba.slice; yubaba.service
# has After=kamaji.service + Wants=kamaji.service, so enabling yubaba
# transitively starts the other two — but bringing each one up explicitly
# here makes the order deterministic and surfaces failure on the right unit.
# yubaba.slice has no [Install] section (slices aren't meant to be enabled —
# see yubaba.slice's own comment), so it gets `start`, not `enable --now`.
- systemctl daemon-reload
- systemctl start yubaba.slice
- systemctl enable --now kamaji.service
- systemctl enable --now yubaba.service