1use crate::stores::ServerStores;
47use nordisk_pairing::elect::Capacity;
48use nordisk_pairing::join::{JoinReply, JoinRequest};
49use nordisk_pairing::pair::{Event, Machine, Step};
50use nordisk_pairing::seed::{Kind, Seed, TunnelKind};
51use nordisk_pairing::signal::Signal;
52use nordisk_pairing::tunnel::{Identity, Tunnel, TunnelError, TunnelStatus};
53use nordisk_pairing::CONTROL_PORT;
54use std::net::Ipv4Addr;
55use std::path::{Path, PathBuf};
56use std::sync::Arc;
57use std::time::Duration;
58
59pub const SEED_REL: &str = "pairing/seed.toml";
61pub const SIGNAL_REL: &str = "pairing/signal.toml";
62pub const JOINED_REL: &str = "pairing/joined.toml";
63pub const CONF_MARK: &str = "# written by gunnar's pairing loop";
66pub const OWN_CERT_REL: &str = "pairing/box.crt";
67pub const OWN_KEY_REL: &str = "pairing/box.key";
68pub const IFACE: &str = "wg0";
70pub const OVERLAY_PREFIX: u8 = 24;
71pub const FEED_ADMIN: &str = "admin_pubkey";
73pub const FEED_RECHARGE_FROM: &str = nordisk_pairing::door::RECHARGE_FROM_FEED;
79pub use crate::twin::RECHARGE_REL;
82
83#[derive(Debug, Clone, Copy, PartialEq, Eq)]
85pub enum Hold {
86 Awaiting,
89 Restoring { head: u64 },
91 Restarting,
94 Restored,
96}
97
98pub fn seed(root: &Path) -> anyhow::Result<Option<Seed>> {
109 seed_with(root, &metadata_user_data, METADATA_TRIES, METADATA_EVERY)
110}
111
112pub const SEED_FROM_METADATA: &str = "nordisk-pairing-seed-from = \"upcloud-metadata\"";
114pub const METADATA_AT: &str = "169.254.169.254:80";
116pub const METADATA_USER_DATA: &str = "/metadata/v1/user_data";
118pub const METADATA_TRIES: u32 = 36;
121pub const METADATA_EVERY: Duration = Duration::from_secs(5);
122
123pub fn seed_with(root: &Path, fetch: &dyn Fn() -> Result<String, String>, tries: u32, every: Duration) -> anyhow::Result<Option<Seed>> {
125 let p = root.join(SEED_REL);
126 let text = match std::fs::read_to_string(&p) {
127 Ok(t) => t,
128 Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None),
129 Err(e) => anyhow::bail!("{}: {e}", p.display()),
130 };
131 if is_metadata_marker(&text) {
132 let mut last = String::new();
133 for n in 0..tries.max(1) {
134 match fetch().and_then(|body| seed_of_user_data(&body)) {
135 Ok(seed) => {
136 let rendered = seed.render().map_err(|e| anyhow::anyhow!("the metadata's seed: {e}"))?;
137 write_secret(root, SEED_REL, &rendered)?;
138 tracing::info!(estate = %seed.estate, "pairing: this box's seed came from the metadata (a release image carries none); kept at {}", p.display());
139 return Ok(Some(seed));
140 }
141 Err(why) => last = why,
142 }
143 if n + 1 < tries {
144 std::thread::sleep(every);
145 }
146 }
147 anyhow::bail!("{}: a release image's seed is the metadata's, and the metadata gave none after {tries} tries: {last}", p.display());
148 }
149 Ok(Some(Seed::parse(&text).map_err(|e| anyhow::anyhow!("{}: {e}", p.display()))?))
150}
151
152pub fn is_metadata_marker(text: &str) -> bool {
154 let lines: Vec<&str> = text.lines().map(str::trim).filter(|l| !l.is_empty() && !l.starts_with('#')).collect();
155 lines == [SEED_FROM_METADATA]
156}
157
158pub fn seed_of_user_data(body: &str) -> Result<Seed, String> {
161 if body.trim().is_empty() {
162 return Err("the server's user_data is empty: it was laid without a seed".into());
163 }
164 Seed::parse(body).map_err(|e| format!("the server's user_data is not a pairing seed: {e}"))
165}
166
167pub fn metadata_user_data() -> Result<String, String> {
169 let at: std::net::SocketAddr = METADATA_AT.parse().map_err(|e| format!("{METADATA_AT}: {e}"))?;
170 match http_get(at, "169.254.169.254", METADATA_USER_DATA)? {
171 (200, body) => Ok(body),
172 (st, body) => Err(format!("the metadata answered HTTP {st}: {}", body.chars().take(200).collect::<String>())),
173 }
174}
175
176pub fn http_get(at: std::net::SocketAddr, host: &str, path: &str) -> Result<(u16, String), String> {
178 use std::io::{Read as _, Write as _};
179 let mut tcp = std::net::TcpStream::connect_timeout(&at, Duration::from_secs(3)).map_err(|e| format!("{at}: {e}"))?;
180 tcp.set_read_timeout(Some(Duration::from_secs(10))).map_err(|e| e.to_string())?;
181 tcp.write_all(format!("GET {path} HTTP/1.0\r\nHost: {host}\r\nAccept: text/plain\r\nConnection: close\r\n\r\n").as_bytes()).map_err(|e| format!("{at}: {e}"))?;
182 let mut raw = Vec::new();
183 tcp.read_to_end(&mut raw).map_err(|e| format!("{at}: {e}"))?;
184 let text = String::from_utf8_lossy(&raw);
185 let (head, body) = text.split_once("\r\n\r\n").ok_or_else(|| format!("{at} answered no HTTP"))?;
186 let status = head.split_whitespace().nth(1).and_then(|s| s.parse().ok()).ok_or_else(|| format!("{at} answered no status: {}", head.lines().next().unwrap_or("")))?;
187 Ok((status, body.to_string()))
188}
189
190pub fn wg_conf(front: &nordisk_pairing::join::Peer, overlay: Ipv4Addr, private_b64: &str, psk: Option<&str>) -> Result<String, String> {
198 let psk_line = match psk.map(str::trim) {
204 Some(p) => {
205 wg_appliance_core::decode_key(p).map_err(|_| "the installer fed a WireGuard pre-shared key that is not one (32 bytes, base64)".to_string())?;
206 format!("PresharedKey = {p}\n")
207 }
208 None => String::new(),
209 };
210 let (net, bits) = nordisk_pairing::OVERLAY_NET;
211 let endpoint = front.endpoint.as_deref().filter(|e| !e.trim().is_empty()).ok_or_else(|| format!("the door fed the front `{}` with no endpoint to dial", front.name))?;
212 if front.tunnel_pubkey.trim().is_empty() {
213 return Err(format!("the door fed the front `{}` with no tunnel key", front.name));
214 }
215 Ok(format!(
216 "{CONF_MARK}: the address is the installer door's answer, the key was minted on this box\n\
217 [Interface]\n\
218 PrivateKey = {private_b64}\n\
219 Address = {overlay}/{OVERLAY_PREFIX}\n\
220 \n\
221 # the front — the one box everybody dials (fed by the door)\n\
222 [Peer]\n\
223 PublicKey = {}\n\
224 {psk_line}\
225 AllowedIPs = {net}/{bits}\n\
226 Endpoint = {endpoint}\n\
227 PersistentKeepalive = 25\n",
228 front.tunnel_pubkey.trim()
229 ))
230}
231
232pub fn fed_psk(root: &Path) -> Option<String> {
237 let text = crate::fed_at_rest::read(root, JOINED_REL)?;
238 let reply = JoinReply::parse(&text).ok()?;
239 reply.feed.get(nordisk_pairing::installer::FEED_WG_PSK).map(|p| p.trim().to_string()).filter(|p| !p.is_empty())
240}
241
242pub fn conf_needs_writing(on_disk: Option<&str>, wanted: &str) -> bool {
248 match on_disk {
249 None => true,
250 Some(text) if text.starts_with(CONF_MARK) => text != wanted,
251 Some(_) => false,
252 }
253}
254
255pub trait Doors {
258 fn join(&self, req: &JoinRequest) -> Result<JoinReply, String>;
260 fn tunnel_up(&self, overlay: Ipv4Addr, peers: &[nordisk_pairing::join::Peer]) -> Result<bool, String>;
263 fn admin(&self, fed: Option<&str>) -> Result<bool, String>;
266 fn become_twin(&self, primary: Ipv4Addr) -> Result<(), String>;
267 fn follow_state(&self) -> Result<(u64, u64), String>;
269 fn recharge_hold(&self, _from: &str) -> Result<Hold, String> {
271 Ok(Hold::Restored)
272 }
273}
274
275pub fn same_primary_utility(reply: Option<&JoinReply>, primary: Ipv4Addr) -> Option<Ipv4Addr> {
281 let reply = reply?;
282 if reply.primary_overlay != Some(primary) {
283 return None;
284 }
285 reply.primary_utility.filter(|u| *u != primary)
286}
287
288pub fn is_fresh(stores: &ServerStores) -> anyhow::Result<bool> {
291 Ok(stores.auth().is_empty()? && stores.auth().bootstrap_window_open()? && stores.catalog().list(None)?.is_empty())
292}
293
294pub fn twin_of(reply: &JoinReply, wg_only: bool) -> Option<String> {
300 if reply.role != Some(nordisk_pairing::Role::Primary) {
301 return None;
302 }
303 let twin = reply.peers.iter().find(|p| p.role == nordisk_pairing::Role::Twin && p.name != reply.me.name)?;
304 let at = crate::tunnel_addr::primary_control(twin.utility, Some(twin.overlay), wg_only)?;
305 Some(format!("{at}:{CONTROL_PORT}"))
306}
307
308pub fn door_role(root: &Path) -> Option<Option<String>> {
311 let text = crate::fed_at_rest::read(root, JOINED_REL)?;
312 Some(JoinReply::parse(&text).ok()?.role.map(|r| r.as_str().to_owned()))
313}
314
315pub fn door_twin(root: &Path) -> Option<String> {
317 let text = crate::fed_at_rest::read(root, JOINED_REL)?;
318 twin_of(&JoinReply::parse(&text).ok()?, crate::tunnel_addr::wg_only())
319}
320
321pub fn recharge_awaited(stores: &ServerStores) -> Option<String> {
325 let from = std::fs::read_to_string(stores.root().join(RECHARGE_REL)).ok()?.trim().to_string();
326 if from.is_empty() {
327 return None;
328 }
329 let root = stores.root();
330 let open = crate::twin::is_restoring(root) || is_fresh(stores).unwrap_or(false);
331 (open && !crate::twin::reseal_pending(root)).then_some(from)
332}
333
334pub fn recharge_hold_of(stores: &ServerStores, from: &str) -> anyhow::Result<Hold> {
336 let root = stores.root();
337 if crate::twin::reseal_pending(root) {
338 return Ok(Hold::Restarting);
339 }
340 if crate::twin::is_restoring(root) {
341 return Ok(Hold::Restoring { head: stores.txlog().head_seq() });
342 }
343 if is_fresh(stores)? {
344 if std::fs::read_to_string(root.join(RECHARGE_REL)).ok().map(|t| t.trim().to_string()).as_deref() != Some(from) {
345 if let Some(dir) = root.join(RECHARGE_REL).parent() {
346 std::fs::create_dir_all(dir)?;
347 }
348 crate::durable::write_atomic_unframed(root, RECHARGE_REL, format!("{from}\n").as_bytes())?;
349 tracing::warn!(recharge_from = %from, "pairing: RECHARGE — this box is a re-admitted main and fresh; it awaits its twin, the admin is held back");
350 }
351 return Ok(Hold::Awaiting);
352 }
353 match std::fs::remove_file(root.join(RECHARGE_REL)) {
354 Ok(()) => tracing::warn!(recharge_from = %from, "pairing: RECHARGE done — this box holds the identities its twin gave back; the ordinary admin step follows"),
355 Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
356 Err(e) => return Err(e.into()),
357 }
358 Ok(Hold::Restored)
359}
360
361pub struct BoxDoors {
363 stores: Arc<ServerStores>,
364 seed: Seed,
365 installed: Arc<crate::installer_listener::Installed>,
369 tunnel: std::sync::Mutex<Option<Result<Box<dyn Tunnel>, TunnelError>>>,
374 feed: std::sync::Mutex<std::collections::BTreeMap<String, String>>,
376}
377
378impl BoxDoors {
379 fn new(stores: Arc<ServerStores>, seed: Seed, installed: Arc<crate::installer_listener::Installed>) -> BoxDoors {
380 let root = stores.root().to_path_buf();
381 let tunnel = (seed.tunnel == TunnelKind::Wireguard).then(|| tunnel_driver(&seed, &root, &Default::default()));
382 BoxDoors { stores, seed, installed, tunnel: std::sync::Mutex::new(tunnel), feed: std::sync::Mutex::new(Default::default()) }
383 }
384
385 fn tunnel_key(&self) -> String {
388 match self.tunnel.lock().ok().as_deref() {
389 Some(Some(Ok(t))) => t.public_key(),
390 _ => String::new(),
391 }
392 }
393}
394
395pub fn write_secret(root: &Path, rel: &str, text: &str) -> anyhow::Result<()> {
396 if let Some(parent) = root.join(rel).parent() {
397 std::fs::create_dir_all(parent)?;
398 }
399 crate::durable::write_atomic_unframed(root, rel, text.as_bytes())?;
400 use std::os::unix::fs::PermissionsExt as _;
401 std::fs::set_permissions(root.join(rel), std::fs::Permissions::from_mode(0o600))?;
402 Ok(())
403}
404
405impl Doors for BoxDoors {
406 fn join(&self, _req: &JoinRequest) -> Result<JoinReply, String> {
410 let reply = self.installed.latest().ok_or_else(|| {
411 "waiting for the installer: it connects to this VM's bootstrap listener and feeds it (name, tunnel address, role, peers)".to_string()
412 })?;
413 if let Ok(mut feed) = self.feed.lock() {
414 feed.clone_from(&reply.feed);
415 }
416 Ok(reply)
417 }
418
419 fn tunnel_up(&self, overlay: Ipv4Addr, peers: &[nordisk_pairing::join::Peer]) -> Result<bool, String> {
420 let mut slot = self.tunnel.lock().map_err(|e| e.to_string())?;
421 if slot.is_none() {
422 let feed = self.feed.lock().map_err(|e| e.to_string())?.clone();
423 *slot = Some(tunnel_driver(&self.seed, self.stores.root(), &feed));
424 }
425 let driver = slot.as_mut().expect("built above").as_mut().map_err(|e| e.to_string())?;
426 let me = Identity { overlay, pubkey: driver.public_key() };
427 driver.up(&me, peers).map_err(|e| e.to_string())?;
428 Ok(driver.status().up)
429 }
430
431 fn admin(&self, fed: Option<&str>) -> Result<bool, String> {
432 let auth = self.stores.auth();
433 if let Some(key) = fed {
434 auth.bootstrap_admin(key).map_err(|e| format!("{e:#}"))?;
435 }
436 let present = auth.is_empty().map(|empty| !empty).map_err(|e| format!("{e:#}"))?;
437 if present {
438 let feed = match self.installed.latest() {
447 Some(reply) => {
448 if let Ok(mut kept) = self.feed.lock() {
449 kept.clone_from(&reply.feed);
450 }
451 reply.feed
452 }
453 None => self.feed.lock().map_err(|e| e.to_string())?.clone(),
454 };
455 for line in crate::fed_principals::install(&self.stores, &feed)? {
456 if !line.ends_with("already there") {
457 tracing::info!("pairing: fed principal {line}");
458 }
459 }
460 }
461 Ok(present)
462 }
463
464 fn recharge_hold(&self, from: &str) -> Result<Hold, String> {
465 let hold = recharge_hold_of(&self.stores, from).map_err(|e| format!("{e:#}"))?;
466 #[cfg(feature = "replication")]
469 if hold == Hold::Awaiting {
470 if let Some(words) = crate::repair::on_awaiting(&self.stores, from) {
471 tracing::info!("pairing: {words}");
472 }
473 }
474 Ok(hold)
475 }
476
477 fn become_twin(&self, primary: Ipv4Addr) -> Result<(), String> {
478 let at = format!("{primary}:{CONTROL_PORT}");
479 if let Some(utility) = same_primary_utility(self.installed.latest().as_ref(), primary) {
480 if crate::twin::onto_tunnel(&self.stores, &format!("{utility}:{CONTROL_PORT}"), &at).map_err(|e| format!("{e:#}"))? {
481 tracing::info!(primary = %at, was = %format!("{utility}:{CONTROL_PORT}"), "pairing: this twin follows the same primary over the tunnel now (joined on its utility address before WireGuard-only)");
482 }
483 }
484 crate::twin::become_twin(&self.stores, &at).map(|_| ()).map_err(|e| format!("{e:#}"))?;
485 crate::twin::start_follower(Arc::clone(&self.stores));
486 Ok(())
487 }
488
489 fn follow_state(&self) -> Result<(u64, u64), String> {
490 match crate::twin::status(self.stores.root()) {
491 Some(s) => Ok((s.applied_seq, s.primary_head_seq)),
492 None => Err("the follower has not reported yet".into()),
493 }
494 }
495}
496
497pub struct GunnarWireGuard {
505 root: PathBuf,
506 front_endpoint: Option<String>,
508}
509
510impl GunnarWireGuard {
511 pub fn new(root: &Path, _seed: &Seed) -> GunnarWireGuard {
512 GunnarWireGuard { root: root.to_path_buf(), front_endpoint: None }
513 }
514}
515
516impl Tunnel for GunnarWireGuard {
517 fn kind(&self) -> TunnelKind {
518 TunnelKind::Wireguard
519 }
520
521 fn public_key(&self) -> String {
524 tunnel_pubkey(&self.root)
525 }
526
527 fn up(&mut self, me: &Identity, peers: &[nordisk_pairing::join::Peer]) -> Result<(), TunnelError> {
531 let failed = |e: anyhow::Error| TunnelError::Failed(format!("{e:#}"));
532 let root = self.root.as_path();
533 let front = peers.iter().find(|p| p.role == nordisk_pairing::Role::Front).ok_or_else(|| {
534 TunnelError::Failed(format!(
535 "{IFACE}: the door fed no front tunnel (its ledger names no [front]); nothing to dial"
536 ))
537 })?;
538 let private_b64 = ensure_tunnel_key(root).map_err(failed)?;
539 let psk = fed_psk(root);
540 let wanted = wg_conf(front, me.overlay, &private_b64, psk.as_deref()).map_err(TunnelError::Failed)?;
541 let on_disk = std::fs::read_to_string(root.join(crate::wg::CONFIG_FILE)).ok();
542 if conf_needs_writing(on_disk.as_deref(), &wanted) {
543 write_secret(root, crate::wg::CONFIG_FILE, &wanted).map_err(failed)?;
545 if psk.is_some() {
546 tracing::info!(front = %front.name, "pairing: wg0 dials the front's relay with the pre-shared key the installer fed (post-quantum hardening of the X25519 handshake)");
547 }
548 }
549 self.front_endpoint = front.endpoint.clone();
550 match crate::wg::raise_from_config(root).map_err(failed)? {
551 true => Ok(()),
552 false => Err(TunnelError::Failed(format!("{IFACE}: no configuration to raise"))),
553 }
554 }
555
556 fn add_peer(&mut self, _peer: &nordisk_pairing::join::Peer) -> Result<(), TunnelError> {
557 Ok(())
558 }
559
560 fn status(&self) -> TunnelStatus {
561 let shown = crate::wg::pairing_tunnel(&self.root);
562 let up = shown.as_ref().map(|(_, _, up)| *up).unwrap_or(false) || crate::wg::config_tunnel_handshaken().is_some();
563 let handshaken = crate::wg::config_tunnel_handshaken().unwrap_or(false);
564 TunnelStatus {
565 kind: TunnelKind::Wireguard,
566 up,
567 reachable: if handshaken { vec!["front".into()] } else { vec![] },
568 detail: match shown.and_then(|(_, a, _)| a) {
569 Some((a, p)) => format!(
570 "{IFACE} {a}/{p} (userspace WireGuard, the front's relay at {})",
571 self.front_endpoint.as_deref().unwrap_or("the endpoint the door fed")
572 ),
573 None => format!("{IFACE}: no address assigned yet"),
574 },
575 }
576 }
577}
578
579pub fn tunnel_driver(seed: &Seed, root: &Path, feed: &std::collections::BTreeMap<String, String>) -> Result<Box<dyn Tunnel>, TunnelError> {
589 match seed.tunnel {
590 TunnelKind::Wireguard => Ok(Box::new(GunnarWireGuard::new(root, seed))),
591 TunnelKind::None => Err(TunnelError::InstallerLinkNone),
592 _ => nordisk_pairing::tunnel::driver_for(seed, feed),
593 }
594}
595
596pub fn ensure_tunnel_key(root: &Path) -> anyhow::Result<String> {
603 if let Ok(k) = std::fs::read_to_string(root.join(crate::wg::KEY_FILE)) {
604 let k = k.trim();
605 if !k.is_empty() {
606 return Ok(k.to_string());
607 }
608 }
609 let (private, _) = crate::wg::generate_keypair();
610 let b64 = wg_appliance_core::encode_key(&private);
611 write_secret(root, crate::wg::KEY_FILE, &format!("{b64}\n"))?;
612 Ok(b64)
613}
614
615pub fn tunnel_pubkey(root: &Path) -> String {
618 match ensure_tunnel_key(root) {
619 Ok(k) => wg_appliance_core::public_from_private(&k).unwrap_or_default(),
620 Err(e) => {
621 tracing::error!(error = %format!("{e:#}"), "pairing: this box cannot mint its tunnel key; the join carries none");
622 String::new()
623 }
624 }
625}
626
627pub fn capacity(root: &Path) -> Option<Capacity> {
629 let m = crate::capacity::MachineFacts::measure();
630 let cores = m.cpus.ok()? as u32;
631 let ram_bytes = m.memory.ok()?.0;
632 let data_bytes = rustix::fs::statvfs(root).map(|s| s.f_blocks * s.f_frsize).unwrap_or(0);
633 let disk_mib_s = match crate::disktest::last() {
634 Some(crate::disktest::Outcome::Done(r)) => (r.seq_write_bps / (1024.0 * 1024.0)) as u32,
635 _ => 0,
636 };
637 Some(Capacity { cores, ram_bytes, data_bytes, disk_mib_s })
638}
639
640fn request(seed: &Seed, root: &Path, csr: &str, tunnel_pubkey: String) -> JoinRequest {
644 JoinRequest {
645 estate: seed.estate.clone(),
646 kind: Kind::Gunnar,
647 tunnel_pubkey,
648 capacity: capacity(root),
649 csr_pem: csr.to_string(),
650 }
651}
652
653#[derive(Debug, Default, Clone)]
655pub struct Learned {
656 pub overlay: Option<Ipv4Addr>,
657 pub peers: Vec<nordisk_pairing::join::Peer>,
659 pub feed_admin: Option<String>,
660 pub name: Option<String>,
661 pub feed: Option<nordisk_pairing::feedbus::Feed>,
664 pub recharge_from: Option<String>,
666}
667
668pub fn turn(m: &mut Machine, learned: &mut Learned, seed: &Seed, req: &JoinRequest, doors: &dyn Doors) -> Step {
671 let step = m.next();
672 let event = match &step {
673 Step::Join | Step::FrontAwaitPeers => match doors.join(req) {
674 Ok(reply) if reply.estate != seed.estate => Event::JoinRefusedForGood(format!("the door is `{}`, this medium is `{}`", reply.estate, seed.estate)),
675 Ok(reply) => {
676 learned.overlay = Some(reply.me.overlay);
677 learned.peers = reply.peers.clone();
678 learned.name = Some(reply.me.name.clone());
679 learned.feed_admin = reply.feed.get(FEED_ADMIN).cloned();
680 learned.feed = Some(reply.feed.clone());
681 let pairs: Vec<(std::net::Ipv4Addr, std::net::Ipv4Addr)> = reply.peers.iter().filter_map(|p| p.utility.map(|u| (u, p.overlay))).collect();
684 learned.recharge_from = reply
685 .feed
686 .get(FEED_RECHARGE_FROM)
687 .map(|t| t.trim().to_string())
688 .filter(|t| !t.is_empty())
689 .map(|t| if crate::tunnel_addr::wg_only() { crate::tunnel_addr::over_the_tunnel(&t, &pairs) } else { t });
690 match reply.role {
691 None => Event::Transient(format!("admitted as `{}`; the election has not spoken yet", reply.me.name)),
692 Some(r) => Event::Joined { role: r, primary: crate::tunnel_addr::primary_control(reply.primary_utility, reply.primary_overlay, crate::tunnel_addr::wg_only()), peers_admitted: reply.peers.len() as u8, peers_expected: seed.peers_expected },
693 }
694 }
695 Err(e) if ["[join-spent]", "[join-forged]", "[join-unknown]", "[join-estate]", "[join-withdrawn]"].iter().any(|w| e.contains(w)) => Event::JoinRefusedForGood(e),
696 Err(e) => Event::Transient(e),
697 },
698 Step::BringTunnelUp => match learned.overlay {
699 Some(_) if seed.tunnel == TunnelKind::Wireguard && !learned.peers.iter().any(|p| p.role == nordisk_pairing::Role::Front) => {
703 match doors.join(req) {
704 Ok(reply) if reply.estate == seed.estate && reply.front().is_some() => {
705 learned.peers = reply.peers.clone();
706 Event::Transient("the door now feeds the front's tunnel; raising it next".into())
707 }
708 Ok(_) => Event::Transient("the door fed no front tunnel yet (its ledger names no [front])".into()),
709 Err(e) => Event::Transient(e),
710 }
711 }
712 None => Event::Transient("no tunnel address yet".into()),
713 Some(o) => match doors.tunnel_up(o, &learned.peers) {
714 Ok(true) => {
715 crate::clock::tunnel_up();
717 Event::TunnelUp
718 }
719 Ok(false) => Event::Transient(format!("the {} tunnel is not up yet", seed.tunnel.as_str())),
720 Err(e) if e.contains("[tunnel-not-built]") || e.contains("[installer-link-none]") => Event::JoinRefusedForGood(e),
723 Err(e) => Event::Transient(e),
724 },
725 },
726 Step::PrimaryCheckAdmin => {
727 let hold = match learned.recharge_from.as_deref() {
731 Some(from) => doors.recharge_hold(from),
732 None => Ok(Hold::Restored),
733 };
734 match hold {
735 Err(e) => Event::Transient(e),
736 Ok(Hold::Awaiting) => Event::Transient(format!(
737 "RECHARGE: this box is the estate's re-admitted main and is FRESH — it awaits its twin at {} (auto-fix fills it when the estate opted in; else the console's [Recharge from twin] or a guardian order; the admin is held back so the bootstrap window stays open)",
738 learned.recharge_from.as_deref().unwrap_or("?")
739 )),
740 Ok(Hold::Restoring { head }) => Event::Transient(format!("RECHARGE: restoring from the twin at {} — the box's log is at seq {head}", learned.recharge_from.as_deref().unwrap_or("?"))),
741 Ok(Hold::Restarting) => Event::Transient("RECHARGE: restored from the twin — the box restarts to open as the primary it was".into()),
742 Ok(Hold::Restored) => match doors.admin(learned.feed_admin.as_deref()) {
743 Ok(true) => Event::AdminPresent,
744 Ok(false) => Event::Transient(format!("no admin on this box: the door fed no `{FEED_ADMIN}`")),
745 Err(e) => Event::Transient(e),
746 },
747 }
748 }
749 Step::TwinBecome { primary } => match doors.become_twin(*primary) {
750 Ok(()) => Event::TwinConferred,
751 Err(e) => Event::Transient(e),
752 },
753 Step::TwinCertify { .. } | Step::TwinFollow { .. } => match doors.follow_state() {
756 Ok((applied, _)) if matches!(step, Step::TwinCertify { .. }) && applied > 0 => Event::TwinCertified,
757 Ok((applied, head)) if matches!(step, Step::TwinFollow { .. }) => Event::Following { applied, head },
758 Ok((_, head)) => Event::Transient(format!("waiting for the front to certify this twin and for the first entry to apply (primary head {head})")),
759 Err(e) => Event::Transient(e),
760 },
761 Step::Nothing => return step,
762 };
763 m.apply(event);
764 step
765}
766
767#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
769pub struct FeedChange {
770 pub clock: bool,
771 pub monetize: bool,
772}
773
774pub fn feed_change(before: Option<&crate::feed::Fed>, now: &crate::feed::Fed) -> FeedChange {
776 let empty = crate::feed::Fed::default();
777 let before = before.unwrap_or(&empty);
778 FeedChange {
779 clock: before.clock_peer != now.clock_peer || before.clock_key != now.clock_key,
780 monetize: before.monetize_pubkey != now.monetize_pubkey,
781 }
782}
783
784pub fn feed_resyncs_the_clock(change: FeedChange, fed_clock: bool, last: Option<estate_clock::report::Outcome>) -> bool {
792 estate_clock::offer::feed_resyncs(change.clock, fed_clock, last)
793}
794
795pub fn take_feed(feed: Option<&nordisk_pairing::feedbus::Feed>) {
807 let Some(feed) = feed else { return };
808 let fed = &crate::feed::Fed::from_feed(feed);
809 let change = feed_change(crate::feed::Fed::current().as_ref(), fed);
810 let resync = feed_resyncs_the_clock(change, fed.clock_peer.is_some(), crate::clock::last_outcome());
811 nordisk_pairing::feedbus::publish(feed.clone());
812 if change == FeedChange::default() && !resync {
813 return;
814 }
815 if change.clock {
816 tracing::info!(clock_peer = ?fed.clock_peer, clock_key = fed.clock_key.is_some(), "pairing: the door fed the clock — it applies from this sync on");
817 crate::clock::fed_arrived();
818 } else if resync {
819 tracing::info!(clock_peer = ?fed.clock_peer, "pairing: the door fed the same clock again and this process has not set its clock right yet — a sync now");
822 crate::clock::fed_arrived();
823 }
824 if change.monetize {
825 let _ = crate::feed::monetize_arrived(fed.monetize_pubkey.as_deref());
829 }
830}
831
832pub fn take_learned_feed(taken: &mut Option<nordisk_pairing::feedbus::Feed>, learned: Option<&nordisk_pairing::feedbus::Feed>) {
841 if learned.is_some() && learned != taken.as_ref() {
842 take_feed(learned);
843 *taken = learned.cloned();
844 }
845}
846
847fn report(root: &Path, seed: &Seed, name: &str, m: &Machine) {
849 let s = Signal::of(name, seed.tunnel, m);
850 tracing::info!("{}", s.line().trim());
851 if let Some(text) = s.render() {
852 let _ = crate::durable::write_atomic_unframed(root, SIGNAL_REL, text.as_bytes());
853 }
854}
855
856pub fn start(stores: Arc<ServerStores>, seed: Seed) {
866 let root = stores.root().to_path_buf();
867 if let Some(cleared) = crate::installer_listener::reset_if_fresh_install(Path::new("/boot"), &root, &|p| crate::narrate::write_esp_file(p, b"the pairing state of the previous installation was cleared\n")) {
868 tracing::info!(cleared = ?cleared, "pairing: the first boot of this installation — it starts UNPAIRED (the data volume's pairing state is cleared)");
869 }
870 let installed = match crate::installer_listener::Installed::open(&root, seed.clone()) {
871 Ok(i) => Arc::new(i.with_stores(Arc::clone(&stores))),
872 Err(e) => {
873 tracing::error!(error = %format!("{e:#}"), "pairing: this box cannot prepare its own key — it will not pair");
874 return;
875 }
876 };
877 crate::installer_listener::start(Arc::clone(&installed));
878 #[cfg(all(feature = "live-app", target_os = "linux"))]
880 crate::live_update::spawn_confirm(crate::readiness::is_ready);
881 #[cfg(feature = "metered")]
884 crate::self_register::start(Arc::clone(&installed), Arc::clone(&stores));
885 std::thread::Builder::new()
886 .name("gunnar-pairing".into())
887 .spawn(move || {
888 let doors = BoxDoors::new(stores, seed.clone(), installed);
890 let mut m = Machine::default();
891 let mut learned = Learned::default();
892 let mut taken = None;
893 loop {
894 let req = request(&seed, &root, "", doors.tunnel_key());
895 let before = m.clone();
896 let step = turn(&mut m, &mut learned, &seed, &req, &doors);
897 take_learned_feed(&mut taken, learned.feed.as_ref());
898 let name = learned.name.clone().unwrap_or_else(|| "?".into());
899 if m != before {
900 report(&root, &seed, &name, &m);
901 }
902 if step == Step::Nothing {
903 if m.paired() {
904 tracing::info!(name = %name, "pairing: PAIRED — the loop rests");
905 } else {
906 tracing::error!(name = %name, "pairing: REFUSED for good — re-seed this box");
907 }
908 return;
909 }
910 std::thread::sleep(Duration::from_secs(m.backoff_s()));
911 }
912 })
913 .expect("spawning the pairing thread");
914}
915
916#[cfg(test)]
917mod tests {
918 use super::*;
919 use nordisk_pairing::mtls;
920 use nordisk_pairing::join::{Assigned, Peer};
921 use nordisk_pairing::Role;
922 use nordisk_pairing::seed::Front;
923 use std::cell::{Cell, RefCell};
924
925 fn a_seed() -> Seed {
926 let ca = mtls::InstallerCa::mint("gunnar.rs", Duration::from_secs(3600)).unwrap();
927 let boot = ca.issue_bootstrap(Duration::from_secs(3600)).unwrap();
928 Seed {
929 estate: "gunnar.rs".into(),
930 tunnel: TunnelKind::Wireguard,
931 front: Front { door: "10.13.8.179:8474".into() },
932 kind: Kind::Gunnar,
933 installer_ca_pem: ca.cert_pem,
934 bootstrap_cert_pem: boot.cert_pem,
935 bootstrap_key_pem: boot.key_pem,
936 peers_expected: 3,
937 mullvad: None,
938 tailscale: None,
939 installer_link: None,
940 }
941 }
942
943 #[test]
948 fn a_release_image_takes_its_seed_from_the_metadata() {
949 use std::os::unix::fs::PermissionsExt as _;
950 let dir = tempfile::tempdir().unwrap();
951 let root = dir.path();
952 std::fs::create_dir_all(root.join("pairing")).unwrap();
953 std::fs::write(root.join(SEED_REL), format!("# a release image: no seed\n{SEED_FROM_METADATA}\n")).unwrap();
954 let want = Seed {
957 installer_link: Some(nordisk_pairing::seed::LinkSeed {
958 installer_pubkey: "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=".into(),
959 vm_private: "ICEiIyQlJicoKSorLC0uLzAxMjM0NTY3ODk6Ozw9Pj8=".into(),
960 port: nordisk_pairing::installer_link::LINK_PORT,
961 }),
962 ..a_seed()
963 };
964 let user_data = want.render().unwrap();
965 let asked = Cell::new(0);
966 let fetch = || -> Result<String, String> {
967 asked.set(asked.get() + 1);
968 if asked.get() < 3 { Err("the network is not up yet".into()) } else { Ok(user_data.clone()) }
969 };
970 let got = seed_with(root, &fetch, 5, Duration::from_millis(1)).unwrap().expect("a seed");
971 assert_eq!((got.estate.as_str(), got.bootstrap_key_pem.as_str()), (want.estate.as_str(), want.bootstrap_key_pem.as_str()));
972 assert_eq!(asked.get(), 3, "asked again while the metadata had none");
973 assert_eq!(std::fs::metadata(root.join(SEED_REL)).unwrap().permissions().mode() & 0o777, 0o600);
974 let never = || -> Result<String, String> { panic!("a kept seed is not asked for again") };
975 assert_eq!(seed_with(root, &never, 1, Duration::ZERO).unwrap().unwrap().estate, want.estate, "the next start reads the kept seed");
976 std::fs::write(root.join(SEED_REL), SEED_FROM_METADATA).unwrap();
978 let empty = || -> Result<String, String> { Ok(String::new()) };
979 let e = seed_with(root, &empty, 2, Duration::ZERO).unwrap_err().to_string();
980 assert!(e.contains("gave none after 2 tries") && e.contains("laid without a seed"), "{e}");
981 assert!(is_metadata_marker(&std::fs::read_to_string(root.join(SEED_REL)).unwrap()));
982 assert!(seed_of_user_data("#cloud-config\nruncmd: [echo hi]\n").unwrap_err().contains("not a pairing seed"));
983 assert!(!is_metadata_marker(&user_data), "a seed is not the marker");
984 }
985
986 #[test]
988 fn the_metadata_is_read_with_one_plain_get() {
989 use std::io::{Read as _, Write as _};
990 let l = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
991 let at = l.local_addr().unwrap();
992 let t = std::thread::spawn(move || {
993 for answer in ["HTTP/1.0 200 OK\r\nContent-Type: text/plain\r\n\r\nSEED TEXT", "HTTP/1.0 404 Not Found\r\n\r\nnone"] {
994 let (mut c, _) = l.accept().unwrap();
995 let mut buf = [0u8; 512];
996 let n = c.read(&mut buf).unwrap();
997 assert!(String::from_utf8_lossy(&buf[..n]).starts_with("GET /metadata/v1/user_data HTTP/1.0\r\nHost: 169.254.169.254"));
998 c.write_all(answer.as_bytes()).unwrap();
999 }
1000 });
1001 assert_eq!(http_get(at, "169.254.169.254", METADATA_USER_DATA).unwrap(), (200, "SEED TEXT".to_string()));
1002 assert_eq!(http_get(at, "169.254.169.254", METADATA_USER_DATA).unwrap().0, 404);
1003 t.join().unwrap();
1004 }
1005
1006 fn req(seed: &Seed) -> JoinRequest {
1007 JoinRequest { estate: seed.estate.clone(), kind: Kind::Gunnar, tunnel_pubkey: "K".into(), capacity: None, csr_pem: String::new() }
1008 }
1009
1010 #[test]
1013 fn the_first_join_request_carries_a_non_empty_tunnel_key_minted_0600() {
1014 use std::os::unix::fs::PermissionsExt as _;
1015 let dir = tempfile::tempdir().unwrap();
1016 let root = dir.path();
1017 assert!(!root.join(crate::wg::KEY_FILE).exists());
1018 let driver = tunnel_driver(&a_seed(), root, &Default::default()).unwrap();
1019 let first = request(&a_seed(), root, "csr", driver.public_key());
1020 assert!(!first.tunnel_pubkey.is_empty(), "the first join must carry a key: {first:?}");
1021 let mode = std::fs::metadata(root.join(crate::wg::KEY_FILE)).unwrap().permissions().mode() & 0o777;
1022 assert_eq!(mode, 0o600);
1023 let private = ensure_tunnel_key(root).unwrap();
1024 assert_eq!(wg_appliance_core::public_from_private(&private).unwrap(), first.tunnel_pubkey, "the join's key is the key on disk");
1025 let again = tunnel_driver(&a_seed(), root, &Default::default()).unwrap();
1026 assert_eq!(request(&a_seed(), root, "", again.public_key()).tunnel_pubkey, first.tunnel_pubkey, "a later join (a reboot) asks with the same key");
1027 let conf = wg_conf(&fed_front(), nordisk_pairing::OVERLAY_BOX_A, &private, None).unwrap();
1028 assert!(conf.contains(&format!("PrivateKey = {private}")), "the tunnel is raised with the key the door recorded");
1029 }
1030
1031 fn fed_front() -> Peer {
1033 nordisk_pairing::join::FrontTunnel { tunnel_pubkey: "xTIBA5rboUvnH4htodjb6e697QjLERt1NAB4mZqp8Dg=".into(), endpoint: "95.111.211.189:51820".into() }.as_peer()
1034 }
1035
1036 #[test]
1037 fn the_config_written_for_the_assigned_address_is_the_shape_the_tunnel_lane_parses() {
1038 let text = wg_conf(&fed_front(), nordisk_pairing::OVERLAY_BOX_A, "gEi6mvh6b5rKDg7UkleGpsJLlA9ueOKyWhjqJqFjWHg=", None).unwrap();
1039 assert!(text.contains("Address = 10.44.0.2/24"), "{text}");
1040 assert!(text.contains("AllowedIPs = 10.44.0.0/24") && text.contains("Endpoint = 95.111.211.189:51820"), "{text}");
1041 assert!(text.contains("PublicKey = xTIBA5rboUvnH4htodjb6e697QjLERt1NAB4mZqp8Dg="), "the front's key is the FED one: {text}");
1042 assert!(!text.contains("ListenPort"), "a gunnar box only dials");
1043 let no_endpoint = Peer { endpoint: None, ..fed_front() };
1044 assert!(wg_conf(&no_endpoint, nordisk_pairing::OVERLAY_BOX_A, "k", None).unwrap_err().contains("no endpoint"));
1045 }
1046
1047 const PSK: &str = "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=";
1048
1049 #[test]
1054 fn the_conf_carries_the_fed_preshared_key_and_the_tunnel_lane_reads_it() {
1055 let private = "gEi6mvh6b5rKDg7UkleGpsJLlA9ueOKyWhjqJqFjWHg=";
1056 let text = wg_conf(&fed_front(), nordisk_pairing::OVERLAY_BOX_A, private, Some(PSK)).unwrap();
1057 assert!(text.contains(&format!("PresharedKey = {PSK}")), "{text}");
1058 let parsed = crate::wg::WgConfig::parse("wg0", &text).unwrap();
1059 assert_eq!(parsed.peers.len(), 1);
1060 assert_eq!(parsed.peers[0].preshared_key, Some(core::array::from_fn(|i| i as u8)), "the front's peer carries it");
1061 let plain = wg_conf(&fed_front(), nordisk_pairing::OVERLAY_BOX_A, private, None).unwrap();
1062 assert!(!plain.contains("PresharedKey"), "none fed, none written: {plain}");
1063 assert!(wg_conf(&fed_front(), nordisk_pairing::OVERLAY_BOX_A, private, Some("not-a-key")).unwrap_err().contains("pre-shared key"));
1064 }
1065
1066 #[test]
1071 fn the_driver_writes_wg0_conf_0600_with_the_psk_the_installer_fed() {
1072 use std::os::unix::fs::PermissionsExt as _;
1073 let dir = tempfile::tempdir().unwrap();
1074 let root = dir.path();
1075 let reply = JoinReply {
1076 estate: "gunnar.rs".into(),
1077 me: Assigned { name: "main".into(), overlay: nordisk_pairing::OVERLAY_BOX_A },
1078 cert_pem: None,
1079 role: Some(Role::Primary),
1080 primary_overlay: None,
1081 primary_utility: None,
1082 peers: vec![fed_front()],
1083 feed: std::collections::BTreeMap::from([(nordisk_pairing::installer::FEED_WG_PSK.to_string(), PSK.to_string())]),
1084 };
1085 crate::fed_at_rest::write(root, JOINED_REL, &reply.render().unwrap()).unwrap();
1086 let mut driver = tunnel_driver(&a_seed(), root, &Default::default()).unwrap();
1087 let _ = driver.up(&Identity { overlay: nordisk_pairing::OVERLAY_BOX_A, pubkey: String::new() }, &reply.peers);
1088 let conf = std::fs::read_to_string(root.join(crate::wg::CONFIG_FILE)).unwrap();
1089 assert!(conf.contains(&format!("PresharedKey = {PSK}")), "{conf}");
1090 let mode = std::fs::metadata(root.join(crate::wg::CONFIG_FILE)).unwrap().permissions().mode() & 0o777;
1091 assert_eq!(mode, 0o600, "wg0.conf holds the private key and the PSK");
1092 }
1093
1094 #[test]
1097 fn our_conf_is_rewritten_for_another_front_and_an_operators_never() {
1098 let now = wg_conf(&fed_front(), nordisk_pairing::OVERLAY_BOX_A, "k", None).unwrap();
1099 let other = Peer { tunnel_pubkey: "AAAABBBBCCCCDDDDEEEEFFFFGGGGHHHHIIIIJJJJKKK=".into(), ..fed_front() };
1100 let before = wg_conf(&other, nordisk_pairing::OVERLAY_BOX_A, "k", None).unwrap();
1101 assert!(conf_needs_writing(None, &now), "none yet");
1102 assert!(!conf_needs_writing(Some(&now), &now), "the same: left alone");
1103 assert!(conf_needs_writing(Some(&before), &now), "ours, for another front: rewritten");
1104 assert!(!conf_needs_writing(Some("[Interface]\nPrivateKey = x\n"), &now), "an operator's: never");
1105 }
1106
1107 #[test]
1110 fn a_wireguard_box_with_no_fed_front_asks_the_door_again_before_raising_the_tunnel() {
1111 struct NoFrontThenFront(Cell<u32>);
1112 impl Doors for NoFrontThenFront {
1113 fn join(&self, _: &JoinRequest) -> Result<JoinReply, String> {
1114 let n = self.0.get() + 1;
1115 self.0.set(n);
1116 let peers = if n == 1 { vec![] } else { vec![fed_front()] };
1117 Ok(JoinReply { estate: "gunnar.rs".into(), me: Assigned { name: "main".into(), overlay: nordisk_pairing::OVERLAY_BOX_A }, cert_pem: None, role: Some(Role::Primary), primary_overlay: None, primary_utility: None, peers, feed: Default::default() })
1118 }
1119 fn tunnel_up(&self, _: Ipv4Addr, peers: &[Peer]) -> Result<bool, String> {
1120 assert!(peers.iter().any(|p| p.role == Role::Front), "the tunnel is raised only with the fed front");
1121 Ok(true)
1122 }
1123 fn admin(&self, _: Option<&str>) -> Result<bool, String> {
1124 Ok(true)
1125 }
1126 fn become_twin(&self, _: Ipv4Addr) -> Result<(), String> {
1127 unreachable!()
1128 }
1129 fn follow_state(&self) -> Result<(u64, u64), String> {
1130 unreachable!()
1131 }
1132 }
1133 let seed = a_seed();
1134 let doors = NoFrontThenFront(Cell::new(0));
1135 let (mut m, mut learned) = (Machine::default(), Learned::default());
1136 let mut steps = Vec::new();
1137 for _ in 0..10 {
1138 let s = turn(&mut m, &mut learned, &seed, &req(&seed), &doors);
1139 if s == Step::Nothing {
1140 break;
1141 }
1142 steps.push(s.name());
1143 }
1144 assert!(m.paired(), "{m:?}");
1145 assert_eq!(doors.0.get(), 2, "joined, then asked again for the front");
1146 assert_eq!(steps, ["join", "tunnel-up", "tunnel-up", "check-admin"], "{steps:?}");
1147 }
1148
1149 struct Fake {
1153 joins: Cell<u32>,
1154 becomes: Cell<u32>,
1155 polls: Cell<u32>,
1156 role: Role,
1157 admin_fed: RefCell<Option<String>>,
1158 tunnel_for: RefCell<Option<Ipv4Addr>>,
1159 }
1160
1161 impl Fake {
1162 fn new(role: Role) -> Fake {
1163 Fake { joins: Cell::new(0), becomes: Cell::new(0), polls: Cell::new(0), role, admin_fed: RefCell::new(None), tunnel_for: RefCell::new(None) }
1164 }
1165 }
1166
1167 impl Doors for Fake {
1168 fn join(&self, _: &JoinRequest) -> Result<JoinReply, String> {
1169 let n = self.joins.get() + 1;
1170 self.joins.set(n);
1171 let (me, overlay) = match self.role {
1172 Role::Twin => ("main", nordisk_pairing::OVERLAY_BOX_A),
1173 _ => ("twin", nordisk_pairing::OVERLAY_BOX_B),
1174 };
1175 let mut feed = std::collections::BTreeMap::new();
1176 feed.insert(FEED_ADMIN.to_string(), "ssh-ed25519 AAAAconsole".to_string());
1177 Ok(JoinReply {
1178 estate: "gunnar.rs".into(),
1179 me: Assigned { name: me.into(), overlay },
1180 cert_pem: None,
1181 role: if n == 1 { None } else { Some(self.role) },
1182 primary_overlay: Some(nordisk_pairing::OVERLAY_BOX_B),
1183 primary_utility: None,
1184 peers: vec![Peer { name: "front".into(), role: Role::Front, tunnel_pubkey: String::new(), overlay: nordisk_pairing::OVERLAY_FRONT, endpoint: None, utility: None }],
1185 feed,
1186 })
1187 }
1188 fn tunnel_up(&self, overlay: Ipv4Addr, _: &[Peer]) -> Result<bool, String> {
1189 *self.tunnel_for.borrow_mut() = Some(overlay);
1190 Ok(true)
1191 }
1192 fn admin(&self, fed: Option<&str>) -> Result<bool, String> {
1193 *self.admin_fed.borrow_mut() = fed.map(str::to_string);
1194 Ok(fed.is_some())
1195 }
1196 fn become_twin(&self, primary: Ipv4Addr) -> Result<(), String> {
1197 assert_eq!(primary, nordisk_pairing::OVERLAY_BOX_B);
1198 let n = self.becomes.get() + 1;
1199 self.becomes.set(n);
1200 if n < 3 { Err("could not be reached".into()) } else { Ok(()) }
1201 }
1202 fn follow_state(&self) -> Result<(u64, u64), String> {
1203 let n = self.polls.get() + 1;
1204 self.polls.set(n);
1205 Ok(match n {
1206 1 => (0, 12),
1207 2 => (5, 12),
1208 _ => (12, 12),
1209 })
1210 }
1211 }
1212
1213 fn run(doors: &Fake) -> (Machine, Learned, Vec<&'static str>) {
1214 let seed = a_seed();
1215 let r = req(&seed);
1216 let (mut m, mut learned) = (Machine::default(), Learned::default());
1217 let mut steps = Vec::new();
1218 for _ in 0..20 {
1219 let s = turn(&mut m, &mut learned, &seed, &r, doors);
1220 if s == Step::Nothing {
1221 break;
1222 }
1223 steps.push(s.name());
1224 }
1225 (m, learned, steps)
1226 }
1227
1228 #[test]
1229 fn a_twin_joins_then_raises_the_tunnel_on_its_assigned_address_then_becomes_twin_and_follows_with_no_admin() {
1230 let doors = Fake::new(Role::Twin);
1231 let (m, learned, steps) = run(&doors);
1232 assert!(m.paired(), "{m:?}");
1233 assert_eq!(steps, ["join", "join", "tunnel-up", "become-twin", "become-twin", "become-twin", "certify", "certify", "follow"], "{steps:?}");
1234 assert_eq!(*doors.tunnel_for.borrow(), Some(nordisk_pairing::OVERLAY_BOX_A), "the address is the door's answer");
1235 assert_eq!(learned.name.as_deref(), Some("main"));
1236 assert!(doors.admin_fed.borrow().is_none(), "a twin must be fresh for BecomeTwin: it installs no admin");
1237 }
1238
1239 #[test]
1244 fn a_primary_enrols_the_fronts_offers_that_arrived_after_its_join_and_before_its_admin() {
1245 use nordisk_pairing::bootstrap::Vm as _;
1246 let data = tempfile::tempdir().unwrap();
1247 let stores = Arc::new(ServerStores::open(data.path()).unwrap());
1248 let installed = Arc::new(crate::installer_listener::Installed::open(data.path(), a_seed()).unwrap().with_stores(Arc::clone(&stores)));
1249 let doors = BoxDoors::new(Arc::clone(&stores), a_seed(), Arc::clone(&installed));
1250 let key = || {
1251 let pem = gunnar_core::mint_ed25519().unwrap();
1252 gunnar_core::ssh_key::PrivateKey::from_openssh(&pem).unwrap().public_key().to_openssh().unwrap()
1253 };
1254 let (op, certify) = (key(), key());
1255 let reply = |feed: Vec<(String, String)>| JoinReply {
1256 estate: "gunnar.rs".into(),
1257 me: Assigned { name: "main".into(), overlay: nordisk_pairing::OVERLAY_BOX_A },
1258 cert_pem: None,
1259 role: Some(Role::Primary),
1260 primary_overlay: None,
1261 primary_utility: None,
1262 peers: vec![],
1263 feed: feed.into_iter().collect(),
1264 };
1265 installed.fed(&reply(vec![(FEED_ADMIN.into(), op.clone())])).unwrap();
1267 doors.join(&req(&a_seed())).unwrap();
1268 installed.fed(&reply(vec![(FEED_ADMIN.into(), op.clone()), (format!("offer.front.{}", gunnar_pairing_door::front::OFFER_CERTIFY_PUBKEY), certify)])).unwrap();
1270 assert!(stores.auth().principal(gunnar_pairing_door::front::CERTIFY_PRINCIPAL).unwrap().is_none());
1271 assert!(doors.admin(Some(&op)).unwrap());
1273 assert!(stores.auth().principal(gunnar_pairing_door::front::CERTIFY_PRINCIPAL).unwrap().is_some(), "front-certify enrolled from the late offer");
1274 }
1275
1276 #[test]
1281 fn an_upgraded_twin_moves_from_its_primarys_utility_address_only_for_the_same_box() {
1282 let (u, o) = (Ipv4Addr::new(10, 13, 12, 118), nordisk_pairing::OVERLAY_BOX_B);
1283 let reply = |overlay: Option<Ipv4Addr>, utility: Option<Ipv4Addr>| JoinReply {
1284 estate: "gunnar.rs".into(),
1285 me: Assigned { name: "vm2".into(), overlay: nordisk_pairing::OVERLAY_BOX_A },
1286 cert_pem: None,
1287 role: Some(Role::Twin),
1288 primary_overlay: overlay,
1289 primary_utility: utility,
1290 peers: vec![],
1291 feed: Default::default(),
1292 };
1293 assert_eq!(same_primary_utility(Some(&reply(Some(o), Some(u))), o), Some(u), "the same primary: its utility address");
1294 assert_eq!(same_primary_utility(Some(&reply(Some(o), Some(u))), u), None, "told the utility address itself: nothing moves");
1295 assert_eq!(same_primary_utility(Some(&reply(Some(Ipv4Addr::new(10, 200, 0, 9)), Some(u))), o), None, "another box's overlay: nothing moves");
1296 assert_eq!(same_primary_utility(Some(&reply(Some(o), None)), o), None, "no utility address named");
1297 assert_eq!(same_primary_utility(None, o), None, "no answer from the door yet");
1298 }
1299
1300 #[test]
1301 fn a_primary_installs_the_fed_admin_key_and_is_paired() {
1302 let doors = Fake::new(Role::Primary);
1303 let (m, _, steps) = run(&doors);
1304 assert!(m.paired(), "{m:?}");
1305 assert_eq!(steps, ["join", "join", "tunnel-up", "check-admin"]);
1306 assert_eq!(doors.admin_fed.borrow().as_deref(), Some("ssh-ed25519 AAAAconsole"), "the admin key came from the door, not the medium");
1307 assert_eq!(doors.becomes.get(), 0, "a primary never becomes a twin");
1308 }
1309
1310 #[test]
1318 fn a_recharged_primary_holds_its_admin_back_until_it_is_restored_and_restarted() {
1319 struct Recharged {
1320 holds: RefCell<Vec<Hold>>,
1321 admin_calls: Cell<u32>,
1322 asked_from: RefCell<Option<String>>,
1323 }
1324 impl Doors for Recharged {
1325 fn join(&self, _: &JoinRequest) -> Result<JoinReply, String> {
1326 let mut feed = std::collections::BTreeMap::new();
1327 feed.insert(FEED_ADMIN.to_string(), "ssh-ed25519 AAAAconsole".to_string());
1328 feed.insert(FEED_RECHARGE_FROM.to_string(), "10.13.8.3:50051".to_string());
1329 Ok(JoinReply {
1330 estate: "gunnar.rs".into(),
1331 me: Assigned { name: "main".into(), overlay: nordisk_pairing::OVERLAY_BOX_A },
1332 cert_pem: None,
1333 role: Some(Role::Primary),
1334 primary_overlay: None,
1335 primary_utility: None,
1336 peers: vec![fed_front()],
1337 feed,
1338 })
1339 }
1340 fn tunnel_up(&self, _: Ipv4Addr, _: &[Peer]) -> Result<bool, String> {
1341 Ok(true)
1342 }
1343 fn admin(&self, _: Option<&str>) -> Result<bool, String> {
1344 self.admin_calls.set(self.admin_calls.get() + 1);
1345 Ok(true)
1346 }
1347 fn become_twin(&self, _: Ipv4Addr) -> Result<(), String> {
1348 unreachable!("a recharged main is the primary it was")
1349 }
1350 fn follow_state(&self) -> Result<(u64, u64), String> {
1351 unreachable!()
1352 }
1353 fn recharge_hold(&self, from: &str) -> Result<Hold, String> {
1354 *self.asked_from.borrow_mut() = Some(from.to_string());
1355 let mut h = self.holds.borrow_mut();
1356 Ok(if h.is_empty() { Hold::Restored } else { h.remove(0) })
1357 }
1358 }
1359 let doors = Recharged {
1360 holds: RefCell::new(vec![Hold::Awaiting, Hold::Awaiting, Hold::Restoring { head: 7 }, Hold::Restarting, Hold::Restored]),
1361 admin_calls: Cell::new(0),
1362 asked_from: RefCell::new(None),
1363 };
1364 let seed = a_seed();
1365 let (mut m, mut learned) = (Machine::default(), Learned::default());
1366 let mut steps = Vec::new();
1367 for _ in 0..20 {
1368 let s = turn(&mut m, &mut learned, &seed, &req(&seed), &doors);
1369 if s == Step::Nothing {
1370 break;
1371 }
1372 steps.push(s.name());
1373 if s.name() == "check-admin" && doors.admin_calls.get() == 0 {
1374 assert!(!m.paired(), "not paired while the recharge holds: {m:?}");
1375 }
1376 }
1377 assert!(m.paired(), "{m:?} {steps:?}");
1378 assert_eq!(doors.admin_calls.get(), 1, "the admin step ran once, after the restore and the restart");
1379 assert_eq!(doors.asked_from.borrow().as_deref(), Some("10.13.8.3:50051"));
1380 assert_eq!(learned.recharge_from.as_deref(), Some("10.13.8.3:50051"));
1381 assert_eq!(steps.iter().filter(|s| **s == "check-admin").count(), 5, "{steps:?}");
1382 }
1383
1384 #[test]
1389 fn a_fresh_box_fed_recharge_from_awaits_with_its_window_open_and_a_restored_one_does_not() {
1390 use nordisk_pairing::bootstrap::Vm as _;
1391 let data = tempfile::tempdir().unwrap();
1392 let stores = Arc::new(ServerStores::open(data.path()).unwrap());
1393 let installed = Arc::new(crate::installer_listener::Installed::open(data.path(), a_seed()).unwrap().with_stores(Arc::clone(&stores)));
1394 let doors = BoxDoors::new(Arc::clone(&stores), a_seed(), Arc::clone(&installed));
1395 let pem = gunnar_core::mint_ed25519().unwrap();
1396 let op = gunnar_core::ssh_key::PrivateKey::from_openssh(&pem).unwrap().public_key().to_openssh().unwrap();
1397 let feed: std::collections::BTreeMap<String, String> = [(FEED_ADMIN.to_string(), op.clone()), (FEED_RECHARGE_FROM.to_string(), "10.13.8.3:50051".to_string())].into_iter().collect();
1398 installed
1399 .fed(&JoinReply { estate: "gunnar.rs".into(), me: Assigned { name: "main".into(), overlay: nordisk_pairing::OVERLAY_BOX_A }, cert_pem: None, role: Some(Role::Primary), primary_overlay: None, primary_utility: None, peers: vec![], feed })
1400 .unwrap();
1401 doors.join(&req(&a_seed())).unwrap();
1402 assert_eq!(doors.recharge_hold("10.13.8.3:50051").unwrap(), Hold::Awaiting);
1403 assert!(stores.auth().bootstrap_window_open().unwrap(), "the window stays open for the restore");
1404 assert_eq!(std::fs::read_to_string(data.path().join(RECHARGE_REL)).unwrap().trim(), "10.13.8.3:50051");
1405 assert_eq!(recharge_awaited(&stores).as_deref(), Some("10.13.8.3:50051"), "what Bootstrap.Status answers");
1406 stores.auth().bootstrap_admin(&op).unwrap();
1407 assert_eq!(doors.recharge_hold("10.13.8.3:50051").unwrap(), Hold::Restored);
1408 assert!(!data.path().join(RECHARGE_REL).exists(), "the marker goes once the box holds its identities");
1409 assert_eq!(recharge_awaited(&stores), None);
1410 }
1411
1412 #[test]
1417 fn the_tunnel_is_chosen_by_the_seed_kind_through_the_trait() {
1418 let dir = tempfile::tempdir().unwrap();
1419 let root = dir.path();
1420 let mut seed = a_seed();
1421 assert_eq!(tunnel_driver(&seed, root, &Default::default()).unwrap().kind(), TunnelKind::Wireguard);
1422
1423 seed.tunnel = TunnelKind::None;
1424 let none = tunnel_driver(&seed, root, &Default::default()).err().expect("`none` is refused");
1425 assert!(none.to_string().starts_with("REFUSED [installer-link-none]"), "{none}");
1426 assert!(!root.join(crate::wg::CONFIG_FILE).exists(), "`none` raises no wg0");
1427
1428 for kind in [TunnelKind::Tailscale, TunnelKind::Mullvad] {
1429 seed.tunnel = kind;
1430 if let Err(e) = tunnel_driver(&seed, root, &Default::default()) {
1431 assert!(e.to_string().starts_with("REFUSED [tunnel-not-built]"), "{e}");
1432 }
1433 }
1434
1435 struct Driven(std::cell::RefCell<Result<Box<dyn Tunnel>, TunnelError>>);
1437 impl Doors for Driven {
1438 fn join(&self, _: &JoinRequest) -> Result<JoinReply, String> {
1439 Ok(JoinReply {
1440 estate: "gunnar.rs".into(),
1441 me: Assigned { name: "main".into(), overlay: nordisk_pairing::OVERLAY_BOX_A },
1442 cert_pem: None,
1443 role: Some(Role::Primary),
1444 primary_overlay: Some(nordisk_pairing::OVERLAY_BOX_A),
1445 primary_utility: None,
1446 peers: vec![],
1447 feed: Default::default(),
1448 })
1449 }
1450 fn tunnel_up(&self, overlay: Ipv4Addr, peers: &[Peer]) -> Result<bool, String> {
1451 let mut t = self.0.borrow_mut();
1452 let t = t.as_mut().map_err(|e| e.to_string())?;
1453 t.up(&Identity { overlay, pubkey: String::new() }, peers).map_err(|e| e.to_string())?;
1454 Ok(t.status().up)
1455 }
1456 fn admin(&self, _: Option<&str>) -> Result<bool, String> {
1457 Ok(true)
1458 }
1459 fn become_twin(&self, _: Ipv4Addr) -> Result<(), String> {
1460 unreachable!()
1461 }
1462 fn follow_state(&self) -> Result<(u64, u64), String> {
1463 unreachable!()
1464 }
1465 }
1466 let run_with = |seed: &Seed, driver: Result<Box<dyn Tunnel>, TunnelError>| {
1467 let doors = Driven(std::cell::RefCell::new(driver));
1468 let (mut m, mut learned) = (Machine::default(), Learned::default());
1469 for _ in 0..10 {
1470 if turn(&mut m, &mut learned, seed, &req(seed), &doors) == Step::Nothing {
1471 break;
1472 }
1473 }
1474 m
1475 };
1476 seed.tunnel = TunnelKind::None;
1477 let refused = run_with(&seed, tunnel_driver(&seed, root, &Default::default()));
1478 assert!(!refused.paired() && refused.next() == Step::Nothing, "`none` never pairs: {refused:?}");
1479 assert!(Signal::of("main", TunnelKind::None, &refused).line().contains("[installer-link-none]"));
1480 seed.tunnel = TunnelKind::Tailscale;
1481 let refused = run_with(&seed, Err(TunnelError::NotBuilt(TunnelKind::Tailscale)));
1482 assert!(!refused.paired() && refused.next() == Step::Nothing, "{refused:?}");
1483 assert!(Signal::of("main", TunnelKind::Tailscale, &refused).line().contains("[tunnel-not-built]"));
1484 }
1485
1486 struct Withdrawn;
1487 impl Doors for Withdrawn {
1488 fn join(&self, _: &JoinRequest) -> Result<JoinReply, String> {
1489 Err("REFUSED [join-withdrawn] every box has joined; the bootstrap certificate is valid during bootstrap only".into())
1490 }
1491 fn tunnel_up(&self, _: Ipv4Addr, _: &[nordisk_pairing::join::Peer]) -> Result<bool, String> {
1492 unreachable!()
1493 }
1494 fn admin(&self, _: Option<&str>) -> Result<bool, String> {
1495 unreachable!()
1496 }
1497 fn become_twin(&self, _: Ipv4Addr) -> Result<(), String> {
1498 unreachable!()
1499 }
1500 fn follow_state(&self) -> Result<(u64, u64), String> {
1501 unreachable!()
1502 }
1503 }
1504
1505 fn a_twin_peer(utility: Ipv4Addr, overlay: Ipv4Addr) -> Peer {
1506 Peer { name: "twin".into(), role: Role::Twin, tunnel_pubkey: String::new(), overlay, endpoint: None, utility: Some(utility) }
1507 }
1508
1509 fn a_mains_reply(peers: Vec<Peer>) -> JoinReply {
1510 JoinReply {
1511 estate: "gunnar.rs".into(),
1512 me: Assigned { name: "main".into(), overlay: nordisk_pairing::OVERLAY_BOX_A },
1513 cert_pem: None,
1514 role: Some(Role::Primary),
1515 primary_overlay: None,
1516 primary_utility: None,
1517 peers,
1518 feed: std::collections::BTreeMap::new(),
1519 }
1520 }
1521
1522 #[test]
1527 fn a_main_finds_its_twins_control_plane_in_the_doors_peer_list() {
1528 let twin = a_twin_peer(Ipv4Addr::new(10, 13, 8, 3), Ipv4Addr::new(10, 44, 0, 3));
1529 let reply = a_mains_reply(vec![fed_front(), twin.clone()]);
1530 assert_eq!(twin_of(&reply, false).as_deref(), Some("10.13.8.3:50051"));
1531 assert_eq!(twin_of(&reply, true).as_deref(), Some("10.44.0.3:50051"), "WireGuard-only: over the tunnel");
1532 let no_utility = Peer { utility: None, ..twin.clone() };
1533 assert_eq!(twin_of(&a_mains_reply(vec![no_utility]), false).as_deref(), Some("10.44.0.3:50051"), "a ledger without utility: the overlay");
1534 assert_eq!(twin_of(&a_mains_reply(vec![fed_front()]), false), None, "no twin named, none known");
1535 assert_eq!(twin_of(&JoinReply { role: Some(Role::Twin), ..reply.clone() }, false), None, "only a main has a twin to wait for");
1536 assert_eq!(twin_of(&JoinReply { role: None, ..reply.clone() }, false), None, "not elected yet");
1537
1538 let dir = tempfile::tempdir().unwrap();
1539 assert_eq!(door_twin(dir.path()), None, "nothing kept, nothing known");
1540 crate::fed_at_rest::write(dir.path(), JOINED_REL, &reply.render().unwrap()).unwrap();
1541 assert_eq!(door_twin(dir.path()), twin_of(&reply, crate::tunnel_addr::wg_only()), "the kept answer");
1542 assert!(door_twin(dir.path()).is_some());
1543 }
1544
1545 #[test]
1546 fn a_withdrawn_bootstrap_ends_the_loop_by_name() {
1547 let seed = a_seed();
1548 let (mut m, mut learned) = (Machine::default(), Learned::default());
1549 turn(&mut m, &mut learned, &seed, &req(&seed), &Withdrawn);
1550 assert_eq!(m.next(), Step::Nothing);
1551 assert!(Signal::of("?", TunnelKind::Wireguard, &m).line().contains("REFUSED for good: REFUSED [join-withdrawn]"));
1552 }
1553}
1554
1555#[cfg(test)]
1556mod feed_tests {
1557 use super::*;
1558 use nordisk_pairing::mtls;
1559 use crate::feed::Fed;
1560
1561 fn fed(m: Option<&str>, peer: Option<&str>, key: Option<&str>) -> Fed {
1562 Fed { monetize_pubkey: m.map(Into::into), clock_peer: peer.map(Into::into), clock_key: key.map(Into::into), ..Fed::default() }
1563 }
1564
1565 #[test]
1573 fn a_repeated_clock_feed_syncs_while_this_process_has_not_set_its_clock_right() {
1574 use estate_clock::report::Outcome;
1575 let peer = fed(None, Some("peer://10.44.0.1:50123"), Some("ssh-ed25519 AAAAk"));
1576 let same = feed_change(Some(&peer), &peer);
1577 assert_eq!(same, FeedChange::default());
1578 for unset in [None, Some(Outcome::NoQuorum), Some(Outcome::Disagree), Some(Outcome::NotConfigured), Some(Outcome::ApplyFailed)] {
1579 assert!(feed_resyncs_the_clock(same, true, unset), "a repeat feed after {unset:?} must sync");
1580 }
1581 for right in [Outcome::Stepped, Outcome::Slewed, Outcome::AlreadyRight] {
1582 assert!(!feed_resyncs_the_clock(same, true, Some(right)), "a clock set right ({right:?}) is not asked again by a repeat");
1583 }
1584 assert!(!feed_resyncs_the_clock(same, false, Some(Outcome::NoQuorum)), "a feed with no clock starts no sync");
1585 assert!(feed_resyncs_the_clock(FeedChange { clock: true, monetize: false }, true, Some(Outcome::AlreadyRight)), "a NEW clock always syncs");
1586 }
1587
1588 #[test]
1589 fn the_first_answer_changes_what_it_carries_and_a_repeat_changes_nothing() {
1590 let first = fed(Some("aa"), Some("peer://10.13.8.179:50123"), Some("ssh-ed25519 AAAAk"));
1591 assert_eq!(feed_change(None, &first), FeedChange { clock: true, monetize: true });
1592 assert_eq!(feed_change(Some(&first), &first), FeedChange::default(), "a re-join with the same feed re-applies nothing");
1593 assert_eq!(feed_change(None, &fed(None, Some("peer://10.13.8.179:50123"), None)), FeedChange { clock: true, monetize: false });
1594 assert_eq!(feed_change(Some(&first), &fed(Some("bb"), Some("peer://10.13.8.179:50123"), Some("ssh-ed25519 AAAAk"))), FeedChange { clock: false, monetize: true });
1595 }
1596
1597 #[test]
1599 fn a_join_learns_the_fed_values() {
1600 struct Feeds;
1601 impl Doors for Feeds {
1602 fn join(&self, _: &JoinRequest) -> Result<JoinReply, String> {
1603 let mut feed = std::collections::BTreeMap::new();
1604 feed.insert(FEED_ADMIN.to_string(), "ssh-ed25519 AAAAconsole".to_string());
1605 feed.insert(crate::feed::FEED_MONETIZE.to_string(), "ab".repeat(32));
1606 feed.insert(crate::feed::FEED_CLOCK_PEER.to_string(), "peer://10.13.8.179:50123".to_string());
1607 feed.insert(crate::feed::FEED_CLOCK_KEY.to_string(), "ssh-ed25519 AAAAk".to_string());
1608 Ok(JoinReply {
1609 estate: "gunnar.rs".into(),
1610 me: nordisk_pairing::join::Assigned { name: "main".into(), overlay: nordisk_pairing::OVERLAY_BOX_A },
1611 cert_pem: None,
1612 role: None,
1613 primary_overlay: None,
1614 primary_utility: None,
1615 peers: vec![],
1616 feed,
1617 })
1618 }
1619 fn tunnel_up(&self, _: Ipv4Addr, _: &[nordisk_pairing::join::Peer]) -> Result<bool, String> { unreachable!() }
1620 fn admin(&self, _: Option<&str>) -> Result<bool, String> { unreachable!() }
1621 fn become_twin(&self, _: Ipv4Addr) -> Result<(), String> { unreachable!() }
1622 fn follow_state(&self) -> Result<(u64, u64), String> { unreachable!() }
1623 }
1624 let seed = tests_seed();
1625 let req = JoinRequest { estate: seed.estate.clone(), kind: Kind::Gunnar, tunnel_pubkey: "K".into(), capacity: None, csr_pem: String::new() };
1626 let (mut m, mut learned) = (Machine::default(), Learned::default());
1627 turn(&mut m, &mut learned, &seed, &req, &Feeds);
1628 assert_eq!(learned.feed.as_ref().map(Fed::from_feed), Some(fed(Some(&"ab".repeat(32)), Some("peer://10.13.8.179:50123"), Some("ssh-ed25519 AAAAk"))));
1629 }
1630
1631 fn tests_seed() -> Seed {
1632 let ca = mtls::InstallerCa::mint("gunnar.rs", Duration::from_secs(3600)).unwrap();
1633 let boot = ca.issue_bootstrap(Duration::from_secs(3600)).unwrap();
1634 Seed {
1635 estate: "gunnar.rs".into(),
1636 tunnel: TunnelKind::Wireguard,
1637 front: nordisk_pairing::seed::Front { door: "10.13.8.179:8474".into() },
1638 kind: Kind::Gunnar,
1639 installer_ca_pem: ca.cert_pem,
1640 bootstrap_cert_pem: boot.cert_pem,
1641 bootstrap_key_pem: boot.key_pem,
1642 peers_expected: 3,
1643 mullvad: None,
1644 tailscale: None,
1645 installer_link: None,
1646 }
1647 }
1648}