use crate::stores::ServerStores;
use nordisk_pairing::elect::Capacity;
use nordisk_pairing::join::{JoinReply, JoinRequest};
use nordisk_pairing::pair::{Event, Machine, Step};
use nordisk_pairing::seed::{Kind, Seed, TunnelKind};
use nordisk_pairing::signal::Signal;
use nordisk_pairing::tunnel::{Identity, Tunnel, TunnelError, TunnelStatus};
use nordisk_pairing::CONTROL_PORT;
use std::net::Ipv4Addr;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::time::Duration;
pub const SEED_REL: &str = "pairing/seed.toml";
pub const SIGNAL_REL: &str = "pairing/signal.toml";
pub const JOINED_REL: &str = "pairing/joined.toml";
pub const CONF_MARK: &str = "# written by gunnar's pairing loop";
pub const OWN_CERT_REL: &str = "pairing/box.crt";
pub const OWN_KEY_REL: &str = "pairing/box.key";
pub const IFACE: &str = "wg0";
pub const OVERLAY_PREFIX: u8 = 24;
pub const FEED_ADMIN: &str = "admin_pubkey";
pub const FEED_RECHARGE_FROM: &str = nordisk_pairing::door::RECHARGE_FROM_FEED;
pub use crate::twin::RECHARGE_REL;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Hold {
Awaiting,
Restoring { head: u64 },
Restarting,
Restored,
}
pub fn seed(root: &Path) -> anyhow::Result<Option<Seed>> {
seed_with(
root,
&nordisk_pairing::seed::metadata::user_data,
METADATA_TRIES,
METADATA_EVERY,
)
}
pub use nordisk_pairing::seed::metadata::{
METADATA_AT, METADATA_EVERY, METADATA_TRIES, METADATA_USER_DATA,
};
pub use nordisk_pairing::seed::SEED_FROM_METADATA;
pub fn seed_with(
root: &Path,
fetch: &dyn Fn() -> Result<String, String>,
tries: u32,
every: Duration,
) -> anyhow::Result<Option<Seed>> {
let p = root.join(SEED_REL);
let seed = nordisk_pairing::seed::metadata::staged(&p, fetch, tries, every)
.map_err(|e| anyhow::anyhow!(e))?;
if let Some(s) = &seed {
tracing::debug!(estate = %s.estate, "pairing: this box's seed, read at {}", p.display());
}
Ok(seed)
}
pub fn wg_conf(
front: &nordisk_pairing::join::Peer,
overlay: Ipv4Addr,
private_b64: &str,
psk: Option<&str>,
) -> Result<String, String> {
let psk_line = match psk.map(str::trim) {
Some(p) => {
wg_appliance_core::decode_key(p).map_err(|_| "the installer fed a WireGuard pre-shared key that is not one (32 bytes, base64)".to_string())?;
format!("PresharedKey = {p}\n")
}
None => String::new(),
};
let (net, bits) = nordisk_pairing::OVERLAY_NET;
let endpoint = front
.endpoint
.as_deref()
.filter(|e| !e.trim().is_empty())
.ok_or_else(|| {
format!(
"the door fed the front `{}` with no endpoint to dial",
front.name
)
})?;
if front.tunnel_pubkey.trim().is_empty() {
return Err(format!(
"the door fed the front `{}` with no tunnel key",
front.name
));
}
Ok(format!(
"{CONF_MARK}: the address is the installer door's answer, the key was minted on this box\n\
[Interface]\n\
PrivateKey = {private_b64}\n\
Address = {overlay}/{OVERLAY_PREFIX}\n\
\n\
# the front — the one box everybody dials (fed by the door)\n\
[Peer]\n\
PublicKey = {}\n\
{psk_line}\
AllowedIPs = {net}/{bits}\n\
Endpoint = {endpoint}\n\
PersistentKeepalive = 25\n",
front.tunnel_pubkey.trim()
))
}
pub fn fed_psk(root: &Path) -> Option<String> {
let text = crate::fed_at_rest::read(root, JOINED_REL)?;
let reply = JoinReply::parse(&text).ok()?;
reply
.feed
.get(nordisk_pairing::installer::FEED_WG_PSK)
.map(|p| p.trim().to_string())
.filter(|p| !p.is_empty())
}
pub fn conf_needs_writing(on_disk: Option<&str>, wanted: &str) -> bool {
match on_disk {
None => true,
Some(text) if text.starts_with(CONF_MARK) => text != wanted,
Some(_) => false,
}
}
pub trait Doors {
fn join(&self, req: &JoinRequest) -> Result<JoinReply, String>;
fn tunnel_up(
&self,
overlay: Ipv4Addr,
peers: &[nordisk_pairing::join::Peer],
) -> Result<bool, String>;
fn admin(&self, fed: Option<&str>) -> Result<bool, String>;
fn become_twin(&self, primary: Ipv4Addr) -> Result<(), String>;
fn follow_state(&self) -> Result<(u64, u64), String>;
fn recharge_hold(&self, _from: &str) -> Result<Hold, String> {
Ok(Hold::Restored)
}
}
pub fn same_primary_utility(reply: Option<&JoinReply>, primary: Ipv4Addr) -> Option<Ipv4Addr> {
let reply = reply?;
if reply.primary_overlay != Some(primary) {
return None;
}
reply.primary_utility.filter(|u| *u != primary)
}
pub fn is_fresh(stores: &ServerStores) -> anyhow::Result<bool> {
Ok(stores.auth().is_empty()?
&& stores.auth().bootstrap_window_open()?
&& stores.catalog().list(None)?.is_empty())
}
pub fn twin_of(reply: &JoinReply, wg_only: bool) -> Option<String> {
if reply.role != Some(nordisk_pairing::Role::Primary) {
return None;
}
let twin = reply
.peers
.iter()
.find(|p| p.role == nordisk_pairing::Role::Twin && p.name != reply.me.name)?;
let at = crate::tunnel_addr::primary_control(twin.utility, Some(twin.overlay), wg_only)?;
Some(format!("{at}:{CONTROL_PORT}"))
}
pub fn door_role(root: &Path) -> Option<Option<String>> {
let text = crate::fed_at_rest::read(root, JOINED_REL)?;
Some(
JoinReply::parse(&text)
.ok()?
.role
.map(|r| r.as_str().to_owned()),
)
}
pub fn door_twin(root: &Path) -> Option<String> {
let text = crate::fed_at_rest::read(root, JOINED_REL)?;
twin_of(
&JoinReply::parse(&text).ok()?,
crate::tunnel_addr::wg_only(),
)
}
pub fn recharge_awaited(stores: &ServerStores) -> Option<String> {
let from = std::fs::read_to_string(stores.root().join(RECHARGE_REL))
.ok()?
.trim()
.to_string();
if from.is_empty() {
return None;
}
let root = stores.root();
let open = crate::twin::is_restoring(root) || is_fresh(stores).unwrap_or(false);
(open && !crate::twin::reseal_pending(root)).then_some(from)
}
pub fn recharge_hold_of(stores: &ServerStores, from: &str) -> anyhow::Result<Hold> {
let root = stores.root();
if crate::twin::reseal_pending(root) {
return Ok(Hold::Restarting);
}
if crate::twin::is_restoring(root) {
return Ok(Hold::Restoring {
head: stores.txlog().head_seq(),
});
}
if is_fresh(stores)? {
if std::fs::read_to_string(root.join(RECHARGE_REL))
.ok()
.map(|t| t.trim().to_string())
.as_deref()
!= Some(from)
{
if let Some(dir) = root.join(RECHARGE_REL).parent() {
std::fs::create_dir_all(dir)?;
}
crate::durable::write_atomic_unframed(
root,
RECHARGE_REL,
format!("{from}\n").as_bytes(),
)?;
tracing::warn!(recharge_from = %from, "pairing: RECHARGE — this box is a re-admitted main and fresh; it awaits its twin, the admin is held back");
}
return Ok(Hold::Awaiting);
}
match std::fs::remove_file(root.join(RECHARGE_REL)) {
Ok(()) => {
tracing::warn!(recharge_from = %from, "pairing: RECHARGE done — this box holds the identities its twin gave back; the ordinary admin step follows")
}
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
Err(e) => return Err(e.into()),
}
Ok(Hold::Restored)
}
pub struct BoxDoors {
stores: Arc<ServerStores>,
seed: Seed,
installed: Arc<crate::installer_listener::Installed>,
tunnel: std::sync::Mutex<Option<Result<Box<dyn Tunnel>, TunnelError>>>,
feed: std::sync::Mutex<std::collections::BTreeMap<String, String>>,
}
impl BoxDoors {
fn new(
stores: Arc<ServerStores>,
seed: Seed,
installed: Arc<crate::installer_listener::Installed>,
) -> BoxDoors {
let root = stores.root().to_path_buf();
let tunnel = (seed.tunnel == TunnelKind::Wireguard)
.then(|| tunnel_driver(&seed, &root, &Default::default()));
BoxDoors {
stores,
seed,
installed,
tunnel: std::sync::Mutex::new(tunnel),
feed: std::sync::Mutex::new(Default::default()),
}
}
fn tunnel_key(&self) -> String {
match self.tunnel.lock().ok().as_deref() {
Some(Some(Ok(t))) => t.public_key(),
_ => String::new(),
}
}
}
pub fn write_secret(root: &Path, rel: &str, text: &str) -> anyhow::Result<()> {
if let Some(parent) = root.join(rel).parent() {
std::fs::create_dir_all(parent)?;
}
crate::durable::write_atomic_unframed(root, rel, text.as_bytes())?;
use std::os::unix::fs::PermissionsExt as _;
std::fs::set_permissions(root.join(rel), std::fs::Permissions::from_mode(0o600))?;
Ok(())
}
impl Doors for BoxDoors {
fn join(&self, _req: &JoinRequest) -> Result<JoinReply, String> {
let reply = self.installed.latest().ok_or_else(|| {
"waiting for the installer: it connects to this VM's bootstrap listener and feeds it (name, tunnel address, role, peers)".to_string()
})?;
if let Ok(mut feed) = self.feed.lock() {
feed.clone_from(&reply.feed);
}
Ok(reply)
}
fn tunnel_up(
&self,
overlay: Ipv4Addr,
peers: &[nordisk_pairing::join::Peer],
) -> Result<bool, String> {
let mut slot = self.tunnel.lock().map_err(|e| e.to_string())?;
if slot.is_none() {
let feed = self.feed.lock().map_err(|e| e.to_string())?.clone();
*slot = Some(tunnel_driver(&self.seed, self.stores.root(), &feed));
}
let driver = slot
.as_mut()
.expect("built above")
.as_mut()
.map_err(|e| e.to_string())?;
let me = Identity {
overlay,
pubkey: driver.public_key(),
};
driver.up(&me, peers).map_err(|e| e.to_string())?;
Ok(driver.status().up)
}
fn admin(&self, fed: Option<&str>) -> Result<bool, String> {
let auth = self.stores.auth();
if let Some(key) = fed {
auth.bootstrap_admin(key).map_err(|e| format!("{e:#}"))?;
}
let present = auth
.is_empty()
.map(|empty| !empty)
.map_err(|e| format!("{e:#}"))?;
if present {
let feed = match self.installed.latest() {
Some(reply) => {
if let Ok(mut kept) = self.feed.lock() {
kept.clone_from(&reply.feed);
}
reply.feed
}
None => self.feed.lock().map_err(|e| e.to_string())?.clone(),
};
for line in crate::fed_principals::install(&self.stores, &feed)? {
if !line.ends_with("already there") {
tracing::info!("pairing: fed principal {line}");
}
}
}
Ok(present)
}
fn recharge_hold(&self, from: &str) -> Result<Hold, String> {
let hold = recharge_hold_of(&self.stores, from).map_err(|e| format!("{e:#}"))?;
#[cfg(feature = "replication")]
if hold == Hold::Awaiting {
if let Some(words) = crate::repair::on_awaiting(&self.stores, from) {
tracing::info!("pairing: {words}");
}
}
Ok(hold)
}
fn become_twin(&self, primary: Ipv4Addr) -> Result<(), String> {
let at = format!("{primary}:{CONTROL_PORT}");
if let Some(utility) = same_primary_utility(self.installed.latest().as_ref(), primary) {
if crate::twin::onto_tunnel(&self.stores, &format!("{utility}:{CONTROL_PORT}"), &at)
.map_err(|e| format!("{e:#}"))?
{
tracing::info!(primary = %at, was = %format!("{utility}:{CONTROL_PORT}"), "pairing: this twin follows the same primary over the tunnel now (joined on its utility address before WireGuard-only)");
}
}
crate::twin::become_twin(&self.stores, &at)
.map(|_| ())
.map_err(|e| format!("{e:#}"))?;
crate::twin::start_follower(Arc::clone(&self.stores));
Ok(())
}
fn follow_state(&self) -> Result<(u64, u64), String> {
match crate::twin::status(self.stores.root()) {
Some(s) => Ok((s.applied_seq, s.primary_head_seq)),
None => Err("the follower has not reported yet".into()),
}
}
}
pub struct GunnarWireGuard {
root: PathBuf,
front_endpoint: Option<String>,
}
impl GunnarWireGuard {
pub fn new(root: &Path, _seed: &Seed) -> GunnarWireGuard {
GunnarWireGuard {
root: root.to_path_buf(),
front_endpoint: None,
}
}
}
impl Tunnel for GunnarWireGuard {
fn kind(&self) -> TunnelKind {
TunnelKind::Wireguard
}
fn public_key(&self) -> String {
tunnel_pubkey(&self.root)
}
fn up(
&mut self,
me: &Identity,
peers: &[nordisk_pairing::join::Peer],
) -> Result<(), TunnelError> {
let failed = |e: anyhow::Error| TunnelError::Failed(format!("{e:#}"));
let root = self.root.as_path();
let front = peers.iter().find(|p| p.role == nordisk_pairing::Role::Front).ok_or_else(|| {
TunnelError::Failed(format!(
"{IFACE}: the door fed no front tunnel (its ledger names no [front]); nothing to dial"
))
})?;
let private_b64 = ensure_tunnel_key(root).map_err(failed)?;
let psk = fed_psk(root);
let wanted = wg_conf(front, me.overlay, &private_b64, psk.as_deref())
.map_err(TunnelError::Failed)?;
let on_disk = std::fs::read_to_string(root.join(crate::wg::CONFIG_FILE)).ok();
if conf_needs_writing(on_disk.as_deref(), &wanted) {
write_secret(root, crate::wg::CONFIG_FILE, &wanted).map_err(failed)?;
if psk.is_some() {
tracing::info!(front = %front.name, "pairing: wg0 dials the front's relay with the pre-shared key the installer fed (post-quantum hardening of the X25519 handshake)");
}
}
self.front_endpoint = front.endpoint.clone();
match crate::wg::raise_from_config(root).map_err(failed)? {
true => Ok(()),
false => Err(TunnelError::Failed(format!(
"{IFACE}: no configuration to raise"
))),
}
}
fn add_peer(&mut self, _peer: &nordisk_pairing::join::Peer) -> Result<(), TunnelError> {
Ok(())
}
fn status(&self) -> TunnelStatus {
let shown = crate::wg::pairing_tunnel(&self.root);
let up = shown.as_ref().map(|(_, _, up)| *up).unwrap_or(false)
|| crate::wg::config_tunnel_handshaken().is_some();
let handshaken = crate::wg::config_tunnel_handshaken().unwrap_or(false);
TunnelStatus {
kind: TunnelKind::Wireguard,
up,
reachable: if handshaken {
vec!["front".into()]
} else {
vec![]
},
detail: match shown.and_then(|(_, a, _)| a) {
Some((a, p)) => format!(
"{IFACE} {a}/{p} (userspace WireGuard, the front's relay at {})",
self.front_endpoint
.as_deref()
.unwrap_or("the endpoint the door fed")
),
None => format!("{IFACE}: no address assigned yet"),
},
}
}
}
pub fn tunnel_driver(
seed: &Seed,
root: &Path,
feed: &std::collections::BTreeMap<String, String>,
) -> Result<Box<dyn Tunnel>, TunnelError> {
match seed.tunnel {
TunnelKind::Wireguard => Ok(Box::new(GunnarWireGuard::new(root, seed))),
TunnelKind::None => Err(TunnelError::InstallerLinkNone),
_ => nordisk_pairing::tunnel::driver_for(seed, feed),
}
}
pub fn ensure_tunnel_key(root: &Path) -> anyhow::Result<String> {
if let Ok(k) = std::fs::read_to_string(root.join(crate::wg::KEY_FILE)) {
let k = k.trim();
if !k.is_empty() {
return Ok(k.to_string());
}
}
let (private, _) = crate::wg::generate_keypair();
let b64 = wg_appliance_core::encode_key(&private);
write_secret(root, crate::wg::KEY_FILE, &format!("{b64}\n"))?;
Ok(b64)
}
pub fn tunnel_pubkey(root: &Path) -> String {
match ensure_tunnel_key(root) {
Ok(k) => wg_appliance_core::public_from_private(&k).unwrap_or_default(),
Err(e) => {
tracing::error!(error = %format!("{e:#}"), "pairing: this box cannot mint its tunnel key; the join carries none");
String::new()
}
}
}
pub fn capacity(root: &Path) -> Option<Capacity> {
let m = crate::capacity::MachineFacts::measure();
let cores = m.cpus.ok()? as u32;
let ram_bytes = m.memory.ok()?.0;
let data_bytes = rustix::fs::statvfs(root)
.map(|s| s.f_blocks * s.f_frsize)
.unwrap_or(0);
let disk_mib_s = match crate::disktest::last() {
Some(crate::disktest::Outcome::Done(r)) => (r.seq_write_bps / (1024.0 * 1024.0)) as u32,
_ => 0,
};
Some(Capacity {
cores,
ram_bytes,
data_bytes,
disk_mib_s,
})
}
fn request(seed: &Seed, root: &Path, csr: &str, tunnel_pubkey: String) -> JoinRequest {
JoinRequest {
estate: seed.estate.clone(),
kind: Kind::Gunnar,
tunnel_pubkey,
capacity: capacity(root),
csr_pem: csr.to_string(),
}
}
#[derive(Debug, Default, Clone)]
pub struct Learned {
pub overlay: Option<Ipv4Addr>,
pub peers: Vec<nordisk_pairing::join::Peer>,
pub feed_admin: Option<String>,
pub name: Option<String>,
pub feed: Option<nordisk_pairing::feedbus::Feed>,
pub recharge_from: Option<String>,
}
pub fn turn(
m: &mut Machine,
learned: &mut Learned,
seed: &Seed,
req: &JoinRequest,
doors: &dyn Doors,
) -> Step {
let step = m.next();
let event = match &step {
Step::Join | Step::FrontAwaitPeers => match doors.join(req) {
Ok(reply) if reply.estate != seed.estate => Event::JoinRefusedForGood(format!("the door is `{}`, this medium is `{}`", reply.estate, seed.estate)),
Ok(reply) => {
learned.overlay = Some(reply.me.overlay);
learned.peers = reply.peers.clone();
learned.name = Some(reply.me.name.clone());
learned.feed_admin = reply.feed.get(FEED_ADMIN).cloned();
learned.feed = Some(reply.feed.clone());
let pairs: Vec<(std::net::Ipv4Addr, std::net::Ipv4Addr)> = reply.peers.iter().filter_map(|p| p.utility.map(|u| (u, p.overlay))).collect();
learned.recharge_from = reply
.feed
.get(FEED_RECHARGE_FROM)
.map(|t| t.trim().to_string())
.filter(|t| !t.is_empty())
.map(|t| if crate::tunnel_addr::wg_only() { crate::tunnel_addr::over_the_tunnel(&t, &pairs) } else { t });
match reply.role {
None => Event::Transient(format!("admitted as `{}`; the election has not spoken yet", reply.me.name)),
Some(r) => Event::Joined { role: r, primary: crate::tunnel_addr::primary_control(reply.primary_utility, reply.primary_overlay, crate::tunnel_addr::wg_only()), peers_admitted: reply.peers.len() as u8, peers_expected: seed.peers_expected },
}
}
Err(e) if ["[join-spent]", "[join-forged]", "[join-unknown]", "[join-estate]", "[join-withdrawn]"].iter().any(|w| e.contains(w)) => Event::JoinRefusedForGood(e),
Err(e) => Event::Transient(e),
},
Step::BringTunnelUp => match learned.overlay {
Some(_) if seed.tunnel == TunnelKind::Wireguard && !learned.peers.iter().any(|p| p.role == nordisk_pairing::Role::Front) => {
match doors.join(req) {
Ok(reply) if reply.estate == seed.estate && reply.front().is_some() => {
learned.peers = reply.peers.clone();
Event::Transient("the door now feeds the front's tunnel; raising it next".into())
}
Ok(_) => Event::Transient("the door fed no front tunnel yet (its ledger names no [front])".into()),
Err(e) => Event::Transient(e),
}
}
None => Event::Transient("no tunnel address yet".into()),
Some(o) => match doors.tunnel_up(o, &learned.peers) {
Ok(true) => {
crate::clock::tunnel_up();
Event::TunnelUp
}
Ok(false) => Event::Transient(format!("the {} tunnel is not up yet", seed.tunnel.as_str())),
Err(e) if e.contains("[tunnel-not-built]") || e.contains("[installer-link-none]") => Event::JoinRefusedForGood(e),
Err(e) => Event::Transient(e),
},
},
Step::PrimaryCheckAdmin => {
let hold = match learned.recharge_from.as_deref() {
Some(from) => doors.recharge_hold(from),
None => Ok(Hold::Restored),
};
match hold {
Err(e) => Event::Transient(e),
Ok(Hold::Awaiting) => Event::Transient(format!(
"RECHARGE: this box is the estate's re-admitted main and is FRESH — it awaits its twin at {} (auto-fix fills it when the estate opted in; else the console's [Recharge from twin] or a guardian order; the admin is held back so the bootstrap window stays open)",
learned.recharge_from.as_deref().unwrap_or("?")
)),
Ok(Hold::Restoring { head }) => Event::Transient(format!("RECHARGE: restoring from the twin at {} — the box's log is at seq {head}", learned.recharge_from.as_deref().unwrap_or("?"))),
Ok(Hold::Restarting) => Event::Transient("RECHARGE: restored from the twin — the box restarts to open as the primary it was".into()),
Ok(Hold::Restored) => match doors.admin(learned.feed_admin.as_deref()) {
Ok(true) => Event::AdminPresent,
Ok(false) => Event::Transient(format!("no admin on this box: the door fed no `{FEED_ADMIN}`")),
Err(e) => Event::Transient(e),
},
}
}
Step::TwinBecome { primary } => match doors.become_twin(*primary) {
Ok(()) => Event::TwinConferred,
Err(e) => Event::Transient(e),
},
Step::TwinCertify { .. } | Step::TwinFollow { .. } => match doors.follow_state() {
Ok((applied, _)) if matches!(step, Step::TwinCertify { .. }) && applied > 0 => Event::TwinCertified,
Ok((applied, head)) if matches!(step, Step::TwinFollow { .. }) => Event::Following { applied, head },
Ok((_, head)) => Event::Transient(format!("waiting for the front to certify this twin and for the first entry to apply (primary head {head})")),
Err(e) => Event::Transient(e),
},
Step::Nothing => return step,
};
m.apply(event);
step
}
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
pub struct FeedChange {
pub clock: bool,
pub monetize: bool,
}
pub fn feed_change(before: Option<&crate::feed::Fed>, now: &crate::feed::Fed) -> FeedChange {
let empty = crate::feed::Fed::default();
let before = before.unwrap_or(&empty);
FeedChange {
clock: before.clock_peer != now.clock_peer || before.clock_key != now.clock_key,
monetize: before.monetize_pubkey != now.monetize_pubkey,
}
}
pub fn feed_resyncs_the_clock(
change: FeedChange,
fed_clock: bool,
last: Option<estate_clock::report::Outcome>,
) -> bool {
estate_clock::offer::feed_resyncs(change.clock, fed_clock, last)
}
pub fn take_feed(feed: Option<&nordisk_pairing::feedbus::Feed>) {
let Some(feed) = feed else { return };
let fed = &crate::feed::Fed::from_feed(feed);
let change = feed_change(crate::feed::Fed::current().as_ref(), fed);
let resync = feed_resyncs_the_clock(
change,
fed.clock_peer.is_some(),
crate::clock::last_outcome(),
);
nordisk_pairing::feedbus::publish(feed.clone());
if change == FeedChange::default() && !resync {
return;
}
if change.clock {
tracing::info!(clock_peer = ?fed.clock_peer, clock_key = fed.clock_key.is_some(), "pairing: the door fed the clock — it applies from this sync on");
crate::clock::fed_arrived();
} else if resync {
tracing::info!(clock_peer = ?fed.clock_peer, "pairing: the door fed the same clock again and this process has not set its clock right yet — a sync now");
crate::clock::fed_arrived();
}
if change.monetize {
let _ = crate::feed::monetize_arrived(fed.monetize_pubkey.as_deref());
}
}
pub fn take_learned_feed(
taken: &mut Option<nordisk_pairing::feedbus::Feed>,
learned: Option<&nordisk_pairing::feedbus::Feed>,
) {
if learned.is_some() && learned != taken.as_ref() {
take_feed(learned);
*taken = learned.cloned();
}
}
fn report(root: &Path, seed: &Seed, name: &str, m: &Machine) {
let s = Signal::of(name, seed.tunnel, m);
tracing::info!("{}", s.line().trim());
if let Some(text) = s.render() {
let _ = crate::durable::write_atomic_unframed(root, SIGNAL_REL, text.as_bytes());
}
}
pub const PAIRING_DUTY: &str = "gunnar-pairing";
pub fn start(stores: Arc<ServerStores>, seed: Seed) {
let root = stores.root().to_path_buf();
if let Some(cleared) =
crate::installer_listener::reset_if_fresh_install(Path::new("/boot"), &root, &|p| {
crate::narrate::write_esp_file(
p,
b"the pairing state of the previous installation was cleared\n",
)
})
{
tracing::info!(cleared = ?cleared, "pairing: the first boot of this installation — it starts UNPAIRED (the data volume's pairing state is cleared)");
}
let installed = match crate::installer_listener::Installed::open(&root, seed.clone()) {
Ok(i) => Arc::new(i.with_stores(Arc::clone(&stores))),
Err(e) => {
tracing::error!(error = %format!("{e:#}"), "pairing: this box cannot prepare its own key — it will not pair");
return;
}
};
crate::installer_listener::start(Arc::clone(&installed));
#[cfg(all(feature = "live-app", target_os = "linux"))]
crate::live_update::spawn_confirm(crate::readiness::is_ready);
let say: nordisk_pairing::duty::Say = Arc::new(|l: &str| tracing::error!("{l}"));
let started = nordisk_pairing::duty::start_with(
PAIRING_DUTY,
Duration::from_secs(30),
say,
move || {
let doors = BoxDoors::new(stores, seed.clone(), installed);
let mut m = Machine::default();
let mut learned = Learned::default();
let mut taken = None;
move || {
let req = request(&seed, &root, "", doors.tunnel_key());
let before = m.clone();
let step = turn(&mut m, &mut learned, &seed, &req, &doors);
take_learned_feed(&mut taken, learned.feed.as_ref());
let name = learned.name.clone().unwrap_or_else(|| "?".into());
if m != before {
report(&root, &seed, &name, &m);
}
if step == Step::Nothing {
if m.paired() {
tracing::info!(name = %name, "pairing: PAIRED — the loop rests");
} else {
tracing::error!(name = %name, "pairing: REFUSED for good — re-seed this box");
}
return nordisk_pairing::duty::Round::Done;
}
nordisk_pairing::duty::Round::After(Duration::from_secs(m.backoff_s()))
}
},
);
if let Err(e) = started {
tracing::error!(error = %e, "ALERT {PAIRING_DUTY} the pairing loop could not start: this box does not pair until it is restarted");
}
}
#[cfg(test)]
mod tests {
#[test]
fn the_pairing_loop_is_a_duty() {
let src = include_str!("pairing.rs");
let code = &src[..src.find("#[cfg(test)]").unwrap()];
assert!(
code.split_whitespace()
.collect::<String>()
.contains("nordisk_pairing::duty::start_with(PAIRING_DUTY"),
"the loop starts as a duty"
);
assert!(
!code.contains(".expect(\"spawning the pairing thread\")"),
"a thread that cannot start is said, never a crash"
);
let main = include_str!("main.rs");
let after = &main[main.find("gunnar_server::run(").unwrap()..];
assert!(
after[..after.find("logqueue::flush").unwrap()]
.contains("nordisk_pairing::duty::stop_all("),
"the binary stops its duties when the server returns"
);
assert!(
after[..after.find("logqueue::flush").unwrap()]
.contains("gunnar_common::jobs::stop_all("),
"the binary stops its jera jobs when the server returns (A32-14)"
);
}
use super::*;
use nordisk_pairing::join::{Assigned, Peer};
use nordisk_pairing::mtls;
use nordisk_pairing::seed::Front;
use nordisk_pairing::Role;
use std::cell::{Cell, RefCell};
fn a_seed() -> Seed {
let ca = mtls::InstallerCa::mint("gunnar.rs", Duration::from_secs(3600)).unwrap();
let boot = ca.issue_bootstrap(Duration::from_secs(3600)).unwrap();
Seed {
estate: "gunnar.rs".into(),
tunnel: TunnelKind::Wireguard,
front: Front {
door: "10.13.8.179:8474".into(),
},
kind: Kind::Gunnar,
installer_ca_pem: ca.cert_pem,
bootstrap_cert_pem: boot.cert_pem,
bootstrap_key_pem: boot.key_pem,
peers_expected: 3,
mullvad: None,
tailscale: None,
installer_link: None,
}
}
#[test]
fn a_release_image_takes_its_seed_from_the_metadata() {
use std::os::unix::fs::PermissionsExt as _;
let dir = tempfile::tempdir().unwrap();
let root = dir.path();
std::fs::create_dir_all(root.join("pairing")).unwrap();
std::fs::write(
root.join(SEED_REL),
format!("# a release image: no seed\n{SEED_FROM_METADATA}\n"),
)
.unwrap();
let want = Seed {
installer_link: Some(serde_json::from_value(serde_json::json!({"installer_pubkey": "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=", "vm_private": "ICEiIyQlJicoKSorLC0uLzAxMjM0NTY3ODk6Ozw9Pj8=", "port": nordisk_pairing::installer_link::LINK_PORT})).unwrap()),
..a_seed()
};
let user_data = want.render().unwrap();
let asked = Cell::new(0);
let fetch = || -> Result<String, String> {
asked.set(asked.get() + 1);
if asked.get() < 3 {
Err("the network is not up yet".into())
} else {
Ok(user_data.clone())
}
};
let got = seed_with(root, &fetch, 5, Duration::from_millis(1))
.unwrap()
.expect("a seed");
assert_eq!(
(got.estate.as_str(), got.bootstrap_key_pem.as_str()),
(want.estate.as_str(), want.bootstrap_key_pem.as_str())
);
assert_eq!(asked.get(), 3, "asked again while the metadata had none");
assert_eq!(
std::fs::metadata(root.join(SEED_REL))
.unwrap()
.permissions()
.mode()
& 0o777,
0o600
);
let never = || -> Result<String, String> { panic!("a kept seed is not asked for again") };
assert_eq!(
seed_with(root, &never, 1, Duration::ZERO)
.unwrap()
.unwrap()
.estate,
want.estate,
"the next start reads the kept seed"
);
std::fs::write(root.join(SEED_REL), SEED_FROM_METADATA).unwrap();
let empty = || -> Result<String, String> { Ok(String::new()) };
let e = seed_with(root, &empty, 2, Duration::ZERO)
.unwrap_err()
.to_string();
assert!(
e.contains("gave none after 2 tries") && e.contains("laid without a seed"),
"{e}"
);
assert!(nordisk_pairing::seed::is_metadata_marker(
&std::fs::read_to_string(root.join(SEED_REL)).unwrap()
));
assert!(nordisk_pairing::seed::metadata::of_user_data(
"#cloud-config\nruncmd: [echo hi]\n"
)
.unwrap_err()
.contains("not a pairing seed"));
assert!(
!nordisk_pairing::seed::is_metadata_marker(&user_data),
"a seed is not the marker"
);
}
#[test]
fn the_seed_reader_is_nordisk_pairings_and_no_copy_is_left() {
let src = include_str!("pairing.rs");
for name in [
"seed_of_user_data",
"metadata_user_data",
"http_get",
"is_metadata_marker",
] {
assert!(
!src.contains(&format!("pub fn {name}(")),
"pairing.rs still carries `{name}`: it is nordisk_pairing::seed::metadata's"
);
}
for name in [
"METADATA_AT",
"METADATA_USER_DATA",
"METADATA_TRIES",
"METADATA_EVERY",
"SEED_FROM_METADATA",
] {
assert!(
!src.contains(&format!("pub const {name}:")),
"pairing.rs still defines `{name}`: it is nordisk-pairing's"
);
}
assert!(
src.contains(&format!("{}::staged(", "nordisk_pairing::seed::metadata")),
"seed_with reads through the shared reader"
);
assert_eq!(METADATA_TRIES, 36);
assert_eq!(METADATA_EVERY, Duration::from_secs(5));
}
fn req(seed: &Seed) -> JoinRequest {
JoinRequest {
estate: seed.estate.clone(),
kind: Kind::Gunnar,
tunnel_pubkey: "K".into(),
capacity: None,
csr_pem: String::new(),
}
}
#[test]
fn the_first_join_request_carries_a_non_empty_tunnel_key_minted_0600() {
use std::os::unix::fs::PermissionsExt as _;
let dir = tempfile::tempdir().unwrap();
let root = dir.path();
assert!(!root.join(crate::wg::KEY_FILE).exists());
let driver = tunnel_driver(&a_seed(), root, &Default::default()).unwrap();
let first = request(&a_seed(), root, "csr", driver.public_key());
assert!(
!first.tunnel_pubkey.is_empty(),
"the first join must carry a key: {first:?}"
);
let mode = std::fs::metadata(root.join(crate::wg::KEY_FILE))
.unwrap()
.permissions()
.mode()
& 0o777;
assert_eq!(mode, 0o600);
let private = ensure_tunnel_key(root).unwrap();
assert_eq!(
wg_appliance_core::public_from_private(&private).unwrap(),
first.tunnel_pubkey,
"the join's key is the key on disk"
);
let again = tunnel_driver(&a_seed(), root, &Default::default()).unwrap();
assert_eq!(
request(&a_seed(), root, "", again.public_key()).tunnel_pubkey,
first.tunnel_pubkey,
"a later join (a reboot) asks with the same key"
);
let conf = wg_conf(&fed_front(), nordisk_pairing::OVERLAY_BOX_A, &private, None).unwrap();
assert!(
conf.contains(&format!("PrivateKey = {private}")),
"the tunnel is raised with the key the door recorded"
);
}
fn fed_front() -> Peer {
nordisk_pairing::join::FrontTunnel {
tunnel_pubkey: "xTIBA5rboUvnH4htodjb6e697QjLERt1NAB4mZqp8Dg=".into(),
endpoint: "95.111.211.189:51820".into(),
}
.as_peer()
}
#[test]
fn the_config_written_for_the_assigned_address_is_the_shape_the_tunnel_lane_parses() {
let text = wg_conf(
&fed_front(),
nordisk_pairing::OVERLAY_BOX_A,
"gEi6mvh6b5rKDg7UkleGpsJLlA9ueOKyWhjqJqFjWHg=",
None,
)
.unwrap();
assert!(text.contains("Address = 10.44.0.2/24"), "{text}");
assert!(
text.contains("AllowedIPs = 10.44.0.0/24")
&& text.contains("Endpoint = 95.111.211.189:51820"),
"{text}"
);
assert!(
text.contains("PublicKey = xTIBA5rboUvnH4htodjb6e697QjLERt1NAB4mZqp8Dg="),
"the front's key is the FED one: {text}"
);
assert!(!text.contains("ListenPort"), "a gunnar box only dials");
let no_endpoint = Peer {
endpoint: None,
..fed_front()
};
assert!(
wg_conf(&no_endpoint, nordisk_pairing::OVERLAY_BOX_A, "k", None)
.unwrap_err()
.contains("no endpoint")
);
}
const PSK: &str = "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=";
#[test]
fn the_conf_carries_the_fed_preshared_key_and_the_tunnel_lane_reads_it() {
let private = "gEi6mvh6b5rKDg7UkleGpsJLlA9ueOKyWhjqJqFjWHg=";
let text = wg_conf(
&fed_front(),
nordisk_pairing::OVERLAY_BOX_A,
private,
Some(PSK),
)
.unwrap();
assert!(text.contains(&format!("PresharedKey = {PSK}")), "{text}");
let parsed = crate::wg::WgConfig::parse("wg0", &text).unwrap();
assert_eq!(parsed.peers.len(), 1);
assert_eq!(
parsed.peers[0].preshared_key,
Some(core::array::from_fn(|i| i as u8)),
"the front's peer carries it"
);
let plain = wg_conf(&fed_front(), nordisk_pairing::OVERLAY_BOX_A, private, None).unwrap();
assert!(
!plain.contains("PresharedKey"),
"none fed, none written: {plain}"
);
assert!(wg_conf(
&fed_front(),
nordisk_pairing::OVERLAY_BOX_A,
private,
Some("not-a-key")
)
.unwrap_err()
.contains("pre-shared key"));
}
#[test]
fn the_driver_writes_wg0_conf_0600_with_the_psk_the_installer_fed() {
use std::os::unix::fs::PermissionsExt as _;
let dir = tempfile::tempdir().unwrap();
let root = dir.path();
let reply = JoinReply {
estate: "gunnar.rs".into(),
me: Assigned {
name: "main".into(),
overlay: nordisk_pairing::OVERLAY_BOX_A,
},
cert_pem: None,
role: Some(Role::Primary),
primary_overlay: None,
primary_utility: None,
peers: vec![fed_front()],
feed: std::collections::BTreeMap::from([(
nordisk_pairing::installer::FEED_WG_PSK.to_string(),
PSK.to_string(),
)]),
};
crate::fed_at_rest::write(root, JOINED_REL, &reply.render().unwrap()).unwrap();
let mut driver = tunnel_driver(&a_seed(), root, &Default::default()).unwrap();
let _ = driver.up(
&Identity {
overlay: nordisk_pairing::OVERLAY_BOX_A,
pubkey: String::new(),
},
&reply.peers,
);
let conf = std::fs::read_to_string(root.join(crate::wg::CONFIG_FILE)).unwrap();
assert!(conf.contains(&format!("PresharedKey = {PSK}")), "{conf}");
let mode = std::fs::metadata(root.join(crate::wg::CONFIG_FILE))
.unwrap()
.permissions()
.mode()
& 0o777;
assert_eq!(mode, 0o600, "wg0.conf holds the private key and the PSK");
}
#[test]
fn our_conf_is_rewritten_for_another_front_and_an_operators_never() {
let now = wg_conf(&fed_front(), nordisk_pairing::OVERLAY_BOX_A, "k", None).unwrap();
let other = Peer {
tunnel_pubkey: "AAAABBBBCCCCDDDDEEEEFFFFGGGGHHHHIIIIJJJJKKK=".into(),
..fed_front()
};
let before = wg_conf(&other, nordisk_pairing::OVERLAY_BOX_A, "k", None).unwrap();
assert!(conf_needs_writing(None, &now), "none yet");
assert!(
!conf_needs_writing(Some(&now), &now),
"the same: left alone"
);
assert!(
conf_needs_writing(Some(&before), &now),
"ours, for another front: rewritten"
);
assert!(
!conf_needs_writing(Some("[Interface]\nPrivateKey = x\n"), &now),
"an operator's: never"
);
}
#[test]
fn a_wireguard_box_with_no_fed_front_asks_the_door_again_before_raising_the_tunnel() {
struct NoFrontThenFront(Cell<u32>);
impl Doors for NoFrontThenFront {
fn join(&self, _: &JoinRequest) -> Result<JoinReply, String> {
let n = self.0.get() + 1;
self.0.set(n);
let peers = if n == 1 { vec![] } else { vec![fed_front()] };
Ok(JoinReply {
estate: "gunnar.rs".into(),
me: Assigned {
name: "main".into(),
overlay: nordisk_pairing::OVERLAY_BOX_A,
},
cert_pem: None,
role: Some(Role::Primary),
primary_overlay: None,
primary_utility: None,
peers,
feed: Default::default(),
})
}
fn tunnel_up(&self, _: Ipv4Addr, peers: &[Peer]) -> Result<bool, String> {
assert!(
peers.iter().any(|p| p.role == Role::Front),
"the tunnel is raised only with the fed front"
);
Ok(true)
}
fn admin(&self, _: Option<&str>) -> Result<bool, String> {
Ok(true)
}
fn become_twin(&self, _: Ipv4Addr) -> Result<(), String> {
unreachable!()
}
fn follow_state(&self) -> Result<(u64, u64), String> {
unreachable!()
}
}
let seed = a_seed();
let doors = NoFrontThenFront(Cell::new(0));
let (mut m, mut learned) = (Machine::default(), Learned::default());
let mut steps = Vec::new();
for _ in 0..10 {
let s = turn(&mut m, &mut learned, &seed, &req(&seed), &doors);
if s == Step::Nothing {
break;
}
steps.push(s.name());
}
assert!(m.paired(), "{m:?}");
assert_eq!(doors.0.get(), 2, "joined, then asked again for the front");
assert_eq!(
steps,
["join", "tunnel-up", "tunnel-up", "check-admin"],
"{steps:?}"
);
}
struct Fake {
joins: Cell<u32>,
becomes: Cell<u32>,
polls: Cell<u32>,
role: Role,
admin_fed: RefCell<Option<String>>,
tunnel_for: RefCell<Option<Ipv4Addr>>,
}
impl Fake {
fn new(role: Role) -> Fake {
Fake {
joins: Cell::new(0),
becomes: Cell::new(0),
polls: Cell::new(0),
role,
admin_fed: RefCell::new(None),
tunnel_for: RefCell::new(None),
}
}
}
impl Doors for Fake {
fn join(&self, _: &JoinRequest) -> Result<JoinReply, String> {
let n = self.joins.get() + 1;
self.joins.set(n);
let (me, overlay) = match self.role {
Role::Twin => ("main", nordisk_pairing::OVERLAY_BOX_A),
_ => ("twin", nordisk_pairing::OVERLAY_BOX_B),
};
let mut feed = std::collections::BTreeMap::new();
feed.insert(
FEED_ADMIN.to_string(),
"ssh-ed25519 AAAAconsole".to_string(),
);
Ok(JoinReply {
estate: "gunnar.rs".into(),
me: Assigned {
name: me.into(),
overlay,
},
cert_pem: None,
role: if n == 1 { None } else { Some(self.role) },
primary_overlay: Some(nordisk_pairing::OVERLAY_BOX_B),
primary_utility: None,
peers: vec![Peer {
name: "front".into(),
role: Role::Front,
tunnel_pubkey: String::new(),
overlay: nordisk_pairing::OVERLAY_FRONT,
endpoint: None,
utility: None,
}],
feed,
})
}
fn tunnel_up(&self, overlay: Ipv4Addr, _: &[Peer]) -> Result<bool, String> {
*self.tunnel_for.borrow_mut() = Some(overlay);
Ok(true)
}
fn admin(&self, fed: Option<&str>) -> Result<bool, String> {
*self.admin_fed.borrow_mut() = fed.map(str::to_string);
Ok(fed.is_some())
}
fn become_twin(&self, primary: Ipv4Addr) -> Result<(), String> {
assert_eq!(primary, nordisk_pairing::OVERLAY_BOX_B);
let n = self.becomes.get() + 1;
self.becomes.set(n);
if n < 3 {
Err("could not be reached".into())
} else {
Ok(())
}
}
fn follow_state(&self) -> Result<(u64, u64), String> {
let n = self.polls.get() + 1;
self.polls.set(n);
Ok(match n {
1 => (0, 12),
2 => (5, 12),
_ => (12, 12),
})
}
}
fn run(doors: &Fake) -> (Machine, Learned, Vec<&'static str>) {
let seed = a_seed();
let r = req(&seed);
let (mut m, mut learned) = (Machine::default(), Learned::default());
let mut steps = Vec::new();
for _ in 0..20 {
let s = turn(&mut m, &mut learned, &seed, &r, doors);
if s == Step::Nothing {
break;
}
steps.push(s.name());
}
(m, learned, steps)
}
#[test]
fn a_twin_joins_then_raises_the_tunnel_on_its_assigned_address_then_becomes_twin_and_follows_with_no_admin(
) {
let doors = Fake::new(Role::Twin);
let (m, learned, steps) = run(&doors);
assert!(m.paired(), "{m:?}");
assert_eq!(
steps,
[
"join",
"join",
"tunnel-up",
"become-twin",
"become-twin",
"become-twin",
"certify",
"certify",
"follow"
],
"{steps:?}"
);
assert_eq!(
*doors.tunnel_for.borrow(),
Some(nordisk_pairing::OVERLAY_BOX_A),
"the address is the door's answer"
);
assert_eq!(learned.name.as_deref(), Some("main"));
assert!(
doors.admin_fed.borrow().is_none(),
"a twin must be fresh for BecomeTwin: it installs no admin"
);
}
#[test]
fn a_primary_enrols_the_fronts_offers_that_arrived_after_its_join_and_before_its_admin() {
use nordisk_pairing::bootstrap::Vm as _;
let data = tempfile::tempdir().unwrap();
let stores = Arc::new(ServerStores::open(data.path()).unwrap());
let installed = Arc::new(
crate::installer_listener::Installed::open(data.path(), a_seed())
.unwrap()
.with_stores(Arc::clone(&stores)),
);
let doors = BoxDoors::new(Arc::clone(&stores), a_seed(), Arc::clone(&installed));
let key = || {
let pem = gunnar_core::mint_ed25519().unwrap();
gunnar_core::ssh_key::PrivateKey::from_openssh(&pem)
.unwrap()
.public_key()
.to_openssh()
.unwrap()
};
let (op, certify) = (key(), key());
let reply = |feed: Vec<(String, String)>| JoinReply {
estate: "gunnar.rs".into(),
me: Assigned {
name: "main".into(),
overlay: nordisk_pairing::OVERLAY_BOX_A,
},
cert_pem: None,
role: Some(Role::Primary),
primary_overlay: None,
primary_utility: None,
peers: vec![],
feed: feed.into_iter().collect(),
};
installed
.fed(&reply(vec![(FEED_ADMIN.into(), op.clone())]))
.unwrap();
doors.join(&req(&a_seed())).unwrap();
installed
.fed(&reply(vec![
(FEED_ADMIN.into(), op.clone()),
(
format!(
"offer.front.{}",
gunnar_pairing_door::front::OFFER_CERTIFY_PUBKEY
),
certify,
),
]))
.unwrap();
assert!(stores
.auth()
.principal(gunnar_pairing_door::front::CERTIFY_PRINCIPAL)
.unwrap()
.is_none());
assert!(doors.admin(Some(&op)).unwrap());
assert!(
stores
.auth()
.principal(gunnar_pairing_door::front::CERTIFY_PRINCIPAL)
.unwrap()
.is_some(),
"front-certify enrolled from the late offer"
);
}
#[test]
fn an_upgraded_twin_moves_from_its_primarys_utility_address_only_for_the_same_box() {
let (u, o) = (
Ipv4Addr::new(10, 13, 12, 118),
nordisk_pairing::OVERLAY_BOX_B,
);
let reply = |overlay: Option<Ipv4Addr>, utility: Option<Ipv4Addr>| JoinReply {
estate: "gunnar.rs".into(),
me: Assigned {
name: "vm2".into(),
overlay: nordisk_pairing::OVERLAY_BOX_A,
},
cert_pem: None,
role: Some(Role::Twin),
primary_overlay: overlay,
primary_utility: utility,
peers: vec![],
feed: Default::default(),
};
assert_eq!(
same_primary_utility(Some(&reply(Some(o), Some(u))), o),
Some(u),
"the same primary: its utility address"
);
assert_eq!(
same_primary_utility(Some(&reply(Some(o), Some(u))), u),
None,
"told the utility address itself: nothing moves"
);
assert_eq!(
same_primary_utility(Some(&reply(Some(Ipv4Addr::new(10, 200, 0, 9)), Some(u))), o),
None,
"another box's overlay: nothing moves"
);
assert_eq!(
same_primary_utility(Some(&reply(Some(o), None)), o),
None,
"no utility address named"
);
assert_eq!(
same_primary_utility(None, o),
None,
"no answer from the door yet"
);
}
#[test]
fn a_primary_installs_the_fed_admin_key_and_is_paired() {
let doors = Fake::new(Role::Primary);
let (m, _, steps) = run(&doors);
assert!(m.paired(), "{m:?}");
assert_eq!(steps, ["join", "join", "tunnel-up", "check-admin"]);
assert_eq!(
doors.admin_fed.borrow().as_deref(),
Some("ssh-ed25519 AAAAconsole"),
"the admin key came from the door, not the medium"
);
assert_eq!(doors.becomes.get(), 0, "a primary never becomes a twin");
}
#[test]
fn a_recharged_primary_holds_its_admin_back_until_it_is_restored_and_restarted() {
struct Recharged {
holds: RefCell<Vec<Hold>>,
admin_calls: Cell<u32>,
asked_from: RefCell<Option<String>>,
}
impl Doors for Recharged {
fn join(&self, _: &JoinRequest) -> Result<JoinReply, String> {
let mut feed = std::collections::BTreeMap::new();
feed.insert(
FEED_ADMIN.to_string(),
"ssh-ed25519 AAAAconsole".to_string(),
);
feed.insert(
FEED_RECHARGE_FROM.to_string(),
"10.13.8.3:50051".to_string(),
);
Ok(JoinReply {
estate: "gunnar.rs".into(),
me: Assigned {
name: "main".into(),
overlay: nordisk_pairing::OVERLAY_BOX_A,
},
cert_pem: None,
role: Some(Role::Primary),
primary_overlay: None,
primary_utility: None,
peers: vec![fed_front()],
feed,
})
}
fn tunnel_up(&self, _: Ipv4Addr, _: &[Peer]) -> Result<bool, String> {
Ok(true)
}
fn admin(&self, _: Option<&str>) -> Result<bool, String> {
self.admin_calls.set(self.admin_calls.get() + 1);
Ok(true)
}
fn become_twin(&self, _: Ipv4Addr) -> Result<(), String> {
unreachable!("a recharged main is the primary it was")
}
fn follow_state(&self) -> Result<(u64, u64), String> {
unreachable!()
}
fn recharge_hold(&self, from: &str) -> Result<Hold, String> {
*self.asked_from.borrow_mut() = Some(from.to_string());
let mut h = self.holds.borrow_mut();
Ok(if h.is_empty() {
Hold::Restored
} else {
h.remove(0)
})
}
}
let doors = Recharged {
holds: RefCell::new(vec![
Hold::Awaiting,
Hold::Awaiting,
Hold::Restoring { head: 7 },
Hold::Restarting,
Hold::Restored,
]),
admin_calls: Cell::new(0),
asked_from: RefCell::new(None),
};
let seed = a_seed();
let (mut m, mut learned) = (Machine::default(), Learned::default());
let mut steps = Vec::new();
for _ in 0..20 {
let s = turn(&mut m, &mut learned, &seed, &req(&seed), &doors);
if s == Step::Nothing {
break;
}
steps.push(s.name());
if s.name() == "check-admin" && doors.admin_calls.get() == 0 {
assert!(!m.paired(), "not paired while the recharge holds: {m:?}");
}
}
assert!(m.paired(), "{m:?} {steps:?}");
assert_eq!(
doors.admin_calls.get(),
1,
"the admin step ran once, after the restore and the restart"
);
assert_eq!(
doors.asked_from.borrow().as_deref(),
Some("10.13.8.3:50051")
);
assert_eq!(learned.recharge_from.as_deref(), Some("10.13.8.3:50051"));
assert_eq!(
steps.iter().filter(|s| **s == "check-admin").count(),
5,
"{steps:?}"
);
}
#[test]
fn a_fresh_box_fed_recharge_from_awaits_with_its_window_open_and_a_restored_one_does_not() {
use nordisk_pairing::bootstrap::Vm as _;
let data = tempfile::tempdir().unwrap();
let stores = Arc::new(ServerStores::open(data.path()).unwrap());
let installed = Arc::new(
crate::installer_listener::Installed::open(data.path(), a_seed())
.unwrap()
.with_stores(Arc::clone(&stores)),
);
let doors = BoxDoors::new(Arc::clone(&stores), a_seed(), Arc::clone(&installed));
let pem = gunnar_core::mint_ed25519().unwrap();
let op = gunnar_core::ssh_key::PrivateKey::from_openssh(&pem)
.unwrap()
.public_key()
.to_openssh()
.unwrap();
let feed: std::collections::BTreeMap<String, String> = [
(FEED_ADMIN.to_string(), op.clone()),
(
FEED_RECHARGE_FROM.to_string(),
"10.13.8.3:50051".to_string(),
),
]
.into_iter()
.collect();
installed
.fed(&JoinReply {
estate: "gunnar.rs".into(),
me: Assigned {
name: "main".into(),
overlay: nordisk_pairing::OVERLAY_BOX_A,
},
cert_pem: None,
role: Some(Role::Primary),
primary_overlay: None,
primary_utility: None,
peers: vec![],
feed,
})
.unwrap();
doors.join(&req(&a_seed())).unwrap();
assert_eq!(
doors.recharge_hold("10.13.8.3:50051").unwrap(),
Hold::Awaiting
);
assert!(
stores.auth().bootstrap_window_open().unwrap(),
"the window stays open for the restore"
);
assert_eq!(
std::fs::read_to_string(data.path().join(RECHARGE_REL))
.unwrap()
.trim(),
"10.13.8.3:50051"
);
assert_eq!(
recharge_awaited(&stores).as_deref(),
Some("10.13.8.3:50051"),
"what Bootstrap.Status answers"
);
stores.auth().bootstrap_admin(&op).unwrap();
assert_eq!(
doors.recharge_hold("10.13.8.3:50051").unwrap(),
Hold::Restored
);
assert!(
!data.path().join(RECHARGE_REL).exists(),
"the marker goes once the box holds its identities"
);
assert_eq!(recharge_awaited(&stores), None);
}
#[test]
fn the_tunnel_is_chosen_by_the_seed_kind_through_the_trait() {
let dir = tempfile::tempdir().unwrap();
let root = dir.path();
let mut seed = a_seed();
assert_eq!(
tunnel_driver(&seed, root, &Default::default())
.unwrap()
.kind(),
TunnelKind::Wireguard
);
seed.tunnel = TunnelKind::None;
let none = tunnel_driver(&seed, root, &Default::default())
.err()
.expect("`none` is refused");
assert!(
none.to_string()
.starts_with("REFUSED [installer-link-none]"),
"{none}"
);
assert!(
!root.join(crate::wg::CONFIG_FILE).exists(),
"`none` raises no wg0"
);
for kind in [TunnelKind::Tailscale, TunnelKind::Mullvad] {
seed.tunnel = kind;
if let Err(e) = tunnel_driver(&seed, root, &Default::default()) {
assert!(
e.to_string().starts_with("REFUSED [tunnel-not-built]"),
"{e}"
);
}
}
struct Driven(std::cell::RefCell<Result<Box<dyn Tunnel>, TunnelError>>);
impl Doors for Driven {
fn join(&self, _: &JoinRequest) -> Result<JoinReply, String> {
Ok(JoinReply {
estate: "gunnar.rs".into(),
me: Assigned {
name: "main".into(),
overlay: nordisk_pairing::OVERLAY_BOX_A,
},
cert_pem: None,
role: Some(Role::Primary),
primary_overlay: Some(nordisk_pairing::OVERLAY_BOX_A),
primary_utility: None,
peers: vec![],
feed: Default::default(),
})
}
fn tunnel_up(&self, overlay: Ipv4Addr, peers: &[Peer]) -> Result<bool, String> {
let mut t = self.0.borrow_mut();
let t = t.as_mut().map_err(|e| e.to_string())?;
t.up(
&Identity {
overlay,
pubkey: String::new(),
},
peers,
)
.map_err(|e| e.to_string())?;
Ok(t.status().up)
}
fn admin(&self, _: Option<&str>) -> Result<bool, String> {
Ok(true)
}
fn become_twin(&self, _: Ipv4Addr) -> Result<(), String> {
unreachable!()
}
fn follow_state(&self) -> Result<(u64, u64), String> {
unreachable!()
}
}
let run_with = |seed: &Seed, driver: Result<Box<dyn Tunnel>, TunnelError>| {
let doors = Driven(std::cell::RefCell::new(driver));
let (mut m, mut learned) = (Machine::default(), Learned::default());
for _ in 0..10 {
if turn(&mut m, &mut learned, seed, &req(seed), &doors) == Step::Nothing {
break;
}
}
m
};
seed.tunnel = TunnelKind::None;
let refused = run_with(&seed, tunnel_driver(&seed, root, &Default::default()));
assert!(
!refused.paired() && refused.next() == Step::Nothing,
"`none` never pairs: {refused:?}"
);
assert!(Signal::of("main", TunnelKind::None, &refused)
.line()
.contains("[installer-link-none]"));
seed.tunnel = TunnelKind::Tailscale;
let refused = run_with(&seed, Err(TunnelError::NotBuilt(TunnelKind::Tailscale)));
assert!(
!refused.paired() && refused.next() == Step::Nothing,
"{refused:?}"
);
assert!(Signal::of("main", TunnelKind::Tailscale, &refused)
.line()
.contains("[tunnel-not-built]"));
}
struct Withdrawn;
impl Doors for Withdrawn {
fn join(&self, _: &JoinRequest) -> Result<JoinReply, String> {
Err("REFUSED [join-withdrawn] every box has joined; the bootstrap certificate is valid during bootstrap only".into())
}
fn tunnel_up(
&self,
_: Ipv4Addr,
_: &[nordisk_pairing::join::Peer],
) -> Result<bool, String> {
unreachable!()
}
fn admin(&self, _: Option<&str>) -> Result<bool, String> {
unreachable!()
}
fn become_twin(&self, _: Ipv4Addr) -> Result<(), String> {
unreachable!()
}
fn follow_state(&self) -> Result<(u64, u64), String> {
unreachable!()
}
}
fn a_twin_peer(utility: Ipv4Addr, overlay: Ipv4Addr) -> Peer {
Peer {
name: "twin".into(),
role: Role::Twin,
tunnel_pubkey: String::new(),
overlay,
endpoint: None,
utility: Some(utility),
}
}
fn a_mains_reply(peers: Vec<Peer>) -> JoinReply {
JoinReply {
estate: "gunnar.rs".into(),
me: Assigned {
name: "main".into(),
overlay: nordisk_pairing::OVERLAY_BOX_A,
},
cert_pem: None,
role: Some(Role::Primary),
primary_overlay: None,
primary_utility: None,
peers,
feed: std::collections::BTreeMap::new(),
}
}
#[test]
fn a_main_finds_its_twins_control_plane_in_the_doors_peer_list() {
let twin = a_twin_peer(Ipv4Addr::new(10, 13, 8, 3), Ipv4Addr::new(10, 44, 0, 3));
let reply = a_mains_reply(vec![fed_front(), twin.clone()]);
assert_eq!(twin_of(&reply, false).as_deref(), Some("10.13.8.3:50051"));
assert_eq!(
twin_of(&reply, true).as_deref(),
Some("10.44.0.3:50051"),
"WireGuard-only: over the tunnel"
);
let no_utility = Peer {
utility: None,
..twin.clone()
};
assert_eq!(
twin_of(&a_mains_reply(vec![no_utility]), false).as_deref(),
Some("10.44.0.3:50051"),
"a ledger without utility: the overlay"
);
assert_eq!(
twin_of(&a_mains_reply(vec![fed_front()]), false),
None,
"no twin named, none known"
);
assert_eq!(
twin_of(
&JoinReply {
role: Some(Role::Twin),
..reply.clone()
},
false
),
None,
"only a main has a twin to wait for"
);
assert_eq!(
twin_of(
&JoinReply {
role: None,
..reply.clone()
},
false
),
None,
"not elected yet"
);
let dir = tempfile::tempdir().unwrap();
assert_eq!(door_twin(dir.path()), None, "nothing kept, nothing known");
crate::fed_at_rest::write(dir.path(), JOINED_REL, &reply.render().unwrap()).unwrap();
assert_eq!(
door_twin(dir.path()),
twin_of(&reply, crate::tunnel_addr::wg_only()),
"the kept answer"
);
assert!(door_twin(dir.path()).is_some());
}
#[test]
fn a_withdrawn_bootstrap_ends_the_loop_by_name() {
let seed = a_seed();
let (mut m, mut learned) = (Machine::default(), Learned::default());
turn(&mut m, &mut learned, &seed, &req(&seed), &Withdrawn);
assert_eq!(m.next(), Step::Nothing);
assert!(Signal::of("?", TunnelKind::Wireguard, &m)
.line()
.contains("REFUSED for good: REFUSED [join-withdrawn]"));
}
}
#[cfg(test)]
mod feed_tests {
use super::*;
use crate::feed::Fed;
use nordisk_pairing::mtls;
fn fed(m: Option<&str>, peer: Option<&str>, key: Option<&str>) -> Fed {
Fed {
monetize_pubkey: m.map(Into::into),
clock_peer: peer.map(Into::into),
clock_key: key.map(Into::into),
..Fed::default()
}
}
#[test]
fn a_repeated_clock_feed_syncs_while_this_process_has_not_set_its_clock_right() {
use estate_clock::report::Outcome;
let peer = fed(
None,
Some("peer://10.44.0.1:50123"),
Some("ssh-ed25519 AAAAk"),
);
let same = feed_change(Some(&peer), &peer);
assert_eq!(same, FeedChange::default());
for unset in [
None,
Some(Outcome::NoQuorum),
Some(Outcome::Disagree),
Some(Outcome::NotConfigured),
Some(Outcome::ApplyFailed),
] {
assert!(
feed_resyncs_the_clock(same, true, unset),
"a repeat feed after {unset:?} must sync"
);
}
for right in [Outcome::Stepped, Outcome::Slewed, Outcome::AlreadyRight] {
assert!(
!feed_resyncs_the_clock(same, true, Some(right)),
"a clock set right ({right:?}) is not asked again by a repeat"
);
}
assert!(
!feed_resyncs_the_clock(same, false, Some(Outcome::NoQuorum)),
"a feed with no clock starts no sync"
);
assert!(
feed_resyncs_the_clock(
FeedChange {
clock: true,
monetize: false
},
true,
Some(Outcome::AlreadyRight)
),
"a NEW clock always syncs"
);
}
#[test]
fn the_first_answer_changes_what_it_carries_and_a_repeat_changes_nothing() {
let first = fed(
Some("aa"),
Some("peer://10.13.8.179:50123"),
Some("ssh-ed25519 AAAAk"),
);
assert_eq!(
feed_change(None, &first),
FeedChange {
clock: true,
monetize: true
}
);
assert_eq!(
feed_change(Some(&first), &first),
FeedChange::default(),
"a re-join with the same feed re-applies nothing"
);
assert_eq!(
feed_change(None, &fed(None, Some("peer://10.13.8.179:50123"), None)),
FeedChange {
clock: true,
monetize: false
}
);
assert_eq!(
feed_change(
Some(&first),
&fed(
Some("bb"),
Some("peer://10.13.8.179:50123"),
Some("ssh-ed25519 AAAAk")
)
),
FeedChange {
clock: false,
monetize: true
}
);
}
#[test]
fn a_join_learns_the_fed_values() {
struct Feeds;
impl Doors for Feeds {
fn join(&self, _: &JoinRequest) -> Result<JoinReply, String> {
let mut feed = std::collections::BTreeMap::new();
feed.insert(
FEED_ADMIN.to_string(),
"ssh-ed25519 AAAAconsole".to_string(),
);
feed.insert(crate::feed::FEED_MONETIZE.to_string(), "ab".repeat(32));
feed.insert(
crate::feed::FEED_CLOCK_PEER.to_string(),
"peer://10.13.8.179:50123".to_string(),
);
feed.insert(
crate::feed::FEED_CLOCK_KEY.to_string(),
"ssh-ed25519 AAAAk".to_string(),
);
Ok(JoinReply {
estate: "gunnar.rs".into(),
me: nordisk_pairing::join::Assigned {
name: "main".into(),
overlay: nordisk_pairing::OVERLAY_BOX_A,
},
cert_pem: None,
role: None,
primary_overlay: None,
primary_utility: None,
peers: vec![],
feed,
})
}
fn tunnel_up(
&self,
_: Ipv4Addr,
_: &[nordisk_pairing::join::Peer],
) -> Result<bool, String> {
unreachable!()
}
fn admin(&self, _: Option<&str>) -> Result<bool, String> {
unreachable!()
}
fn become_twin(&self, _: Ipv4Addr) -> Result<(), String> {
unreachable!()
}
fn follow_state(&self) -> Result<(u64, u64), String> {
unreachable!()
}
}
let seed = tests_seed();
let req = JoinRequest {
estate: seed.estate.clone(),
kind: Kind::Gunnar,
tunnel_pubkey: "K".into(),
capacity: None,
csr_pem: String::new(),
};
let (mut m, mut learned) = (Machine::default(), Learned::default());
turn(&mut m, &mut learned, &seed, &req, &Feeds);
assert_eq!(
learned.feed.as_ref().map(Fed::from_feed),
Some(fed(
Some(&"ab".repeat(32)),
Some("peer://10.13.8.179:50123"),
Some("ssh-ed25519 AAAAk")
))
);
}
fn tests_seed() -> Seed {
let ca = mtls::InstallerCa::mint("gunnar.rs", Duration::from_secs(3600)).unwrap();
let boot = ca.issue_bootstrap(Duration::from_secs(3600)).unwrap();
Seed {
estate: "gunnar.rs".into(),
tunnel: TunnelKind::Wireguard,
front: nordisk_pairing::seed::Front {
door: "10.13.8.179:8474".into(),
},
kind: Kind::Gunnar,
installer_ca_pem: ca.cert_pem,
bootstrap_cert_pem: boot.cert_pem,
bootstrap_key_pem: boot.key_pem,
peers_expected: 3,
mullvad: None,
tailscale: None,
installer_link: None,
}
}
}