mod netns;
use std::net::Ipv4Addr;
use std::os::fd::RawFd;
use std::sync::Arc;
use std::time::Duration;
use ntk_common::{Cost, HCoord, Naddr, Topology};
use ntk_neighborhood::NodeId;
use ntk_netlink::{Interface, Nexthop, RouteSpec, RouteTarget};
use ntkd::kernel::addressing;
use ntkd::node::lifecycle::PreformedNetwork;
use tokio::sync::Barrier;
use tokio::task::JoinSet;
use tokio_util::sync::CancellationToken;
use netns::{Member, NamespaceWorker, NodeReport, Segment};
fn init_tracing() {
let filter = tracing_subscriber::EnvFilter::try_from_default_env()
.unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info"));
let _ = tracing_subscriber::fmt()
.with_env_filter(filter)
.with_test_writer()
.with_thread_names(true)
.try_init();
}
async fn wait_until(mut check: impl FnMut() -> bool, timeout: Duration) -> bool {
let deadline = tokio::time::Instant::now() + timeout;
loop {
if check() {
return true;
}
if tokio::time::Instant::now() >= deadline {
return check();
}
tokio::time::sleep(Duration::from_millis(100)).await;
}
}
fn cost_at(snapshot: &ntk_qspn::RouteSnapshot, level: usize, pos: u32) -> Option<Cost> {
snapshot
.levels
.get(level)?
.iter()
.find(|e| e.destination == HCoord::new(level, pos))?
.paths
.first()
.map(|p| p.cost)
}
const CHAIN_GSIZES: [u32; 1] = [4];
const CHAIN_PORT: u16 = 27310;
const CHAIN_TIMEOUT: Duration = Duration::from_secs(40);
fn chain_topology() -> Topology {
Topology::new(CHAIN_GSIZES).unwrap()
}
fn chain_devs(idx: u32) -> Vec<&'static str> {
match idx {
0 | 3 => vec!["eth0"],
1 | 2 => vec!["eth0", "eth1"],
_ => unreachable!("chain has exactly 4 nodes"),
}
}
fn chain_dev_toward(node: u32, next_hop: u32) -> &'static str {
match (node, next_hop) {
(0, 1) | (1, 0) | (2, 1) | (3, 2) => "eth0",
(1, 2) | (2, 3) => "eth1",
_ => unreachable!("not a chain edge: {node} -> {next_hop}"),
}
}
fn chain_next_hop(from: u32, to: u32) -> u32 {
if to > from { from + 1 } else { from - 1 }
}
async fn chain_worker_body(
idx: u32,
devs: Vec<&'static str>,
barrier: Arc<Barrier>,
) -> anyhow::Result<NodeReport> {
let dev_index = netns::bring_up_devs(&devs).await?;
let mut tasks = JoinSet::new();
let cancel = CancellationToken::new();
let started = netns::spawn_node(
NodeId::from_raw(301 + idx as i32).unwrap(),
Some(vec![idx]),
None,
&CHAIN_GSIZES,
&devs,
CHAIN_PORT,
&mut tasks,
cancel.clone(),
)
.await?;
let converged = wait_until(
|| {
let snapshot = started.running.generation.borrow().qspn.snapshot();
(0..4u32).all(|j| {
j == idx || {
let hops = u64::from((idx as i32 - j as i32).unsigned_abs());
cost_at(&snapshot, 0, j) == Some(Cost::Finite(hops * netns::RTT_MS))
}
})
},
CHAIN_TIMEOUT,
)
.await;
anyhow::ensure!(
converged,
"node{idx}: did not converge to the exact expected hop-by-hop cost set in time: {:#?}",
started.running.generation.borrow().qspn.snapshot()
);
let report = netns::observe(&format!("node{idx}"), &started, dev_index).await?;
let _ = tokio::time::timeout(CHAIN_TIMEOUT + Duration::from_secs(5), barrier.wait()).await;
netns::teardown(&started, cancel, &mut tasks).await;
Ok(report)
}
#[ignore = "requires the equivalent of CAP_NET_ADMIN over its own network namespaces"]
#[tokio::test]
async fn chain_of_four_converges_to_exact_multi_hop_routes() {
init_tracing();
let barrier = Arc::new(Barrier::new(4));
let workers: Vec<NamespaceWorker<NodeReport>> = (0..4u32)
.map(|i| {
let b = barrier.clone();
let devs = chain_devs(i);
NamespaceWorker::spawn(format!("chain-node{i}"), move || {
chain_worker_body(i, devs, b)
})
})
.collect();
let fds: Vec<RawFd> = workers.iter().map(NamespaceWorker::fd).collect();
let (root, root_driver) = netns::root_handle_with_driver().expect("root rtnetlink handle");
let segments = vec![
Segment {
name: "s0",
members: vec![
Member {
node: 0,
dev: "eth0",
},
Member {
node: 1,
dev: "eth0",
},
],
},
Segment {
name: "s1",
members: vec![
Member {
node: 1,
dev: "eth1",
},
Member {
node: 2,
dev: "eth0",
},
],
},
Segment {
name: "s2",
members: vec![
Member {
node: 2,
dev: "eth1",
},
Member {
node: 3,
dev: "eth0",
},
],
},
];
let mesh = netns::wire(&root, &segments, &fds)
.await
.expect("wire chain segments");
for w in &workers {
w.signal_moved();
}
let results = netns::join_all(workers, CHAIN_TIMEOUT + netns::JOIN_MARGIN).await;
netns::teardown_mesh(mesh, root, root_driver).await;
let reports: Vec<NodeReport> = results
.into_iter()
.enumerate()
.map(|(i, r)| r.unwrap_or_else(|e| panic!("chain worker {i} failed: {e:?}")))
.collect();
let topo = chain_topology();
for (idx, devs) in [(1u32, ["eth0", "eth1"]), (2u32, ["eth0", "eth1"])] {
let report = &reports[idx as usize];
for dev in devs {
assert!(
report.arc_cost(dev).is_some(),
"node{idx}'s NIC {dev:?} never completed an arc (no published cost) — the \
relay node must have a working arc on *every* configured NIC, not just the \
first: {:#?}",
report.arcs
);
}
}
for i in 0..4u32 {
let report = &reports[i as usize];
assert_eq!(
report.routes.len(),
3,
"node{i} must have exactly the 3 converged destinations installed, nothing stale: {:#?}",
report.routes
);
for j in 0..4u32 {
if i == j {
continue;
}
let next_hop = chain_next_hop(i, j);
let dev = chain_dev_toward(i, next_hop);
let via = reports[next_hop as usize]
.linklocal(chain_dev_toward(next_hop, i))
.unwrap_or_else(|| {
panic!("node{next_hop} has no linklocal address on its link toward node{i}")
});
let expected = RouteSpec {
destination: addressing::gnode_destination(
&Naddr::new(topo.clone(), vec![i]).unwrap(),
HCoord::new(0, j),
)
.unwrap(),
table: report.route_table,
target: RouteTarget::Gateway {
via,
dev: Interface::Index(*report.dev_index.get(dev).unwrap()),
src: Some(
addressing::host_address(&Naddr::new(topo.clone(), vec![i]).unwrap())
.unwrap()
.address(),
),
},
};
assert!(
report.routes.contains(&expected),
"node{i}'s route to node{j} (via node{next_hop}) is not exactly {expected:?}: {:#?}",
report.routes
);
}
}
}
const LEVEL1_GSIZES: [u32; 2] = [2, 2];
const LEVEL1_PORT: u16 = 27320;
const LEVEL1_TIMEOUT: Duration = Duration::from_secs(40);
fn level1_position(idx: u32) -> Vec<u32> {
match idx {
0 => vec![0, 0],
1 => vec![0, 1],
2 => vec![1, 1],
_ => unreachable!("level1 mesh has exactly 3 nodes"),
}
}
async fn level1_worker_body(idx: u32, barrier: Arc<Barrier>) -> anyhow::Result<NodeReport> {
let devs = ["eth0"];
let dev_index = netns::bring_up_devs(&devs).await?;
let mut tasks = JoinSet::new();
let cancel = CancellationToken::new();
let started = netns::spawn_node(
NodeId::from_raw(401 + idx as i32).unwrap(),
Some(level1_position(idx)),
None,
&LEVEL1_GSIZES,
&devs,
LEVEL1_PORT,
&mut tasks,
cancel.clone(),
)
.await?;
let converged = wait_until(
|| {
let snapshot = started.running.generation.borrow().qspn.snapshot();
match idx {
0 => snapshot
.levels
.get(1)
.and_then(|level| level.iter().find(|e| e.destination == HCoord::new(1, 1)))
.is_some_and(|e| {
e.paths.len() == 2
&& e.paths
.iter()
.all(|p| p.cost == Cost::Finite(netns::RTT_MS))
}),
1 => {
cost_at(&snapshot, 0, 1) == Some(Cost::Finite(netns::RTT_MS))
&& cost_at(&snapshot, 1, 0) == Some(Cost::Finite(netns::RTT_MS))
}
2 => {
cost_at(&snapshot, 0, 0) == Some(Cost::Finite(netns::RTT_MS))
&& cost_at(&snapshot, 1, 0) == Some(Cost::Finite(netns::RTT_MS))
}
_ => unreachable!(),
}
},
LEVEL1_TIMEOUT,
)
.await;
anyhow::ensure!(
converged,
"node{idx}: did not converge to the expected level-0/level-1 cost set in time: {:#?}",
started.running.generation.borrow().qspn.snapshot()
);
if idx == 1 || idx == 2 {
let snapshot = started.running.generation.borrow().qspn.snapshot();
for level in 0..2 {
if let Some(entries) = snapshot.levels.get(level) {
for entry in entries {
let arc_ids: Vec<_> = entry.paths.iter().map(|p| p.arc).collect();
eprintln!(
"DIAG node{idx} dest={:?} admitted_path_arcs={arc_ids:?}",
entry.destination
);
}
}
}
}
let report = netns::observe(&format!("node{idx}"), &started, dev_index).await?;
let _ = tokio::time::timeout(LEVEL1_TIMEOUT + Duration::from_secs(5), barrier.wait()).await;
netns::teardown(&started, cancel, &mut tasks).await;
Ok(report)
}
#[ignore = "requires the equivalent of CAP_NET_ADMIN over its own network namespaces"]
#[tokio::test]
async fn level1_destination_installs_correct_cidr_route() {
init_tracing();
let barrier = Arc::new(Barrier::new(3));
let workers: Vec<NamespaceWorker<NodeReport>> = (0..3u32)
.map(|i| {
let b = barrier.clone();
NamespaceWorker::spawn(format!("level1-node{i}"), move || level1_worker_body(i, b))
})
.collect();
let fds: Vec<RawFd> = workers.iter().map(NamespaceWorker::fd).collect();
let (root, root_driver) = netns::root_handle_with_driver().expect("root rtnetlink handle");
let segments = vec![Segment {
name: "seg",
members: vec![
Member {
node: 0,
dev: "eth0",
},
Member {
node: 1,
dev: "eth0",
},
Member {
node: 2,
dev: "eth0",
},
],
}];
let mesh = netns::wire(&root, &segments, &fds)
.await
.expect("wire level1 segment");
for w in &workers {
w.signal_moved();
}
let results = netns::join_all(workers, LEVEL1_TIMEOUT + netns::JOIN_MARGIN).await;
netns::teardown_mesh(mesh, root, root_driver).await;
let reports: Vec<NodeReport> = results
.into_iter()
.enumerate()
.map(|(i, r)| r.unwrap_or_else(|e| panic!("level1 worker {i} failed: {e:?}")))
.collect();
let topo = Topology::new(LEVEL1_GSIZES).unwrap();
let naddr = |idx: u32| Naddr::new(topo.clone(), level1_position(idx)).unwrap();
let gateway = |idx: u32| Nexthop {
via: reports[idx as usize]
.linklocal("eth0")
.unwrap_or_else(|| panic!("node{idx} has no linklocal address on eth0")),
dev: Interface::Index(*reports[idx as usize].dev_index.get("eth0").unwrap()),
weight: 255,
};
let assert_valid_nexthops = |report: &NodeReport, idx: u32, hc: HCoord, neighbours: &[u32]| {
let expected_dest = addressing::gnode_destination(&naddr(idx), hc).unwrap();
let route = report
.routes
.iter()
.find(|r| r.destination == expected_dest)
.unwrap_or_else(|| panic!("node{idx} has no route to {hc:?}: {:#?}", report.routes));
let vias: Vec<Ipv4Addr> = match &route.target {
RouteTarget::Gateway { via, .. } => vec![*via],
RouteTarget::Multipath(nexthops) => nexthops.iter().map(|n| n.via).collect(),
other => panic!("node{idx}'s route to {hc:?} is not routable: {other:?}"),
};
let unique: std::collections::HashSet<_> = vias.iter().collect();
assert_eq!(
unique.len(),
vias.len(),
"node{idx}'s route to {hc:?} repeats a gateway — a nexthop set can never legitimately \
name the same neighbour twice: {vias:?}"
);
let allowed: Vec<Ipv4Addr> = neighbours.iter().map(|&n| gateway(n).via).collect();
for via in &vias {
assert!(
allowed.contains(via),
"node{idx}'s route to {hc:?} names {via}, not one of this segment's real \
neighbours {allowed:?}"
);
}
};
assert_valid_nexthops(&reports[1], 1, HCoord::new(0, 1), &[2]);
assert_valid_nexthops(&reports[1], 1, HCoord::new(1, 0), &[0, 2]);
assert_valid_nexthops(&reports[2], 2, HCoord::new(0, 0), &[1]);
assert_valid_nexthops(&reports[2], 2, HCoord::new(1, 0), &[0, 1]);
assert_eq!(reports[0].routes.len(), 1, "{:#?}", reports[0].routes);
let route = &reports[0].routes[0];
let expected_dest = addressing::gnode_destination(&naddr(0), HCoord::new(1, 1)).unwrap();
assert_eq!(route.destination, expected_dest);
assert_eq!(route.table, reports[0].route_table);
let RouteTarget::Multipath(nexthops) = &route.target else {
panic!(
"p0's route to the level-1 slot-1 aggregate must be Multipath (two disjoint equal-cost \
paths via p1 and p2), got {:?}",
route.target
);
};
let vias: std::collections::HashSet<_> = nexthops.iter().map(|n| n.via).collect();
assert_eq!(
vias.len(),
nexthops.len(),
"p0's level-1 route repeats a gateway: {nexthops:?}"
);
assert_eq!(
vias,
std::collections::HashSet::from([gateway(1).via, gateway(2).via]),
"p0's level-1 route must name exactly p1 and p2, its two disjoint one-hop neighbours: \
{nexthops:?}"
);
}
const PARTITION_GSIZES: [u32; 2] = [2, 2];
const PARTITION_PORT: u16 = 27330;
const PARTITION_TIMEOUT: Duration = Duration::from_secs(60);
const SEVERANCE_GSIZES: [u32; 2] = [2, 2];
const SEVERANCE_PORT: u16 = 27335;
const SEVERANCE_TIMEOUT: Duration = Duration::from_secs(40);
fn severance_position(idx: u32) -> Vec<u32> {
match idx {
0 => vec![0, 0],
1 => vec![1, 0],
2 => vec![0, 1],
3 => vec![1, 1],
_ => unreachable!("severance mesh has exactly 4 nodes"),
}
}
async fn severance_worker_body(
idx: u32,
sever_barrier: Arc<Barrier>,
done_barrier: Arc<Barrier>,
) -> anyhow::Result<NodeReport> {
let devs = ["eth0"];
let dev_index = netns::bring_up_devs(&devs).await?;
let mut tasks = JoinSet::new();
let cancel = CancellationToken::new();
let started = netns::spawn_node(
NodeId::from_raw(511 + idx as i32).unwrap(),
Some(severance_position(idx)),
None,
&SEVERANCE_GSIZES,
&devs,
SEVERANCE_PORT,
&mut tasks,
cancel.clone(),
)
.await?;
let sibling_pos0 = if idx.is_multiple_of(2) {
idx + 1
} else {
idx - 1
};
let other_slot = 1 - severance_position(idx)[1];
let converged = wait_until(
|| {
let snapshot = started.running.generation.borrow().qspn.snapshot();
let two_disjoint_paths_to_other_slot = snapshot
.levels
.get(1)
.and_then(|level| {
level
.iter()
.find(|e| e.destination == HCoord::new(1, other_slot))
})
.is_some_and(|e| {
e.paths
.iter()
.filter(|p| p.cost == Cost::Finite(netns::RTT_MS))
.count()
== 2
});
cost_at(&snapshot, 0, sibling_pos0) == Some(Cost::Finite(netns::RTT_MS))
&& two_disjoint_paths_to_other_slot
},
SEVERANCE_TIMEOUT,
)
.await;
anyhow::ensure!(
converged,
"node{idx}: did not converge to its sibling plus both disjoint paths to the other slot: \
{:#?}",
started.running.generation.borrow().qspn.snapshot()
);
let _ = tokio::time::timeout(
SEVERANCE_TIMEOUT + Duration::from_secs(5),
sever_barrier.wait(),
)
.await;
let dropped_exactly_the_unreachable_destination = wait_until(
|| {
let snapshot = started.running.generation.borrow().qspn.snapshot();
let other_slot_gone = snapshot.levels.get(1).is_none_or(|level| {
!level
.iter()
.any(|e| e.destination == HCoord::new(1, other_slot))
});
cost_at(&snapshot, 0, sibling_pos0) == Some(Cost::Finite(netns::RTT_MS))
&& other_slot_gone
},
Duration::from_secs(20),
)
.await;
anyhow::ensure!(
dropped_exactly_the_unreachable_destination,
"node{idx}: did not drop exactly the unreachable other-slot destination within 20s: {:#?}",
started.running.generation.borrow().qspn.snapshot()
);
let report = netns::observe(&format!("node{idx}"), &started, dev_index).await?;
let _ = tokio::time::timeout(
SEVERANCE_TIMEOUT + Duration::from_secs(5),
done_barrier.wait(),
)
.await;
netns::teardown(&started, cancel, &mut tasks).await;
Ok(report)
}
#[ignore = "requires the equivalent of CAP_NET_ADMIN over its own network namespaces"]
#[tokio::test]
async fn partition_clean_severance_drops_exactly_the_unreachable_destinations() {
init_tracing();
let sever_barrier = Arc::new(Barrier::new(5));
let done_barrier = Arc::new(Barrier::new(5));
let workers: Vec<NamespaceWorker<NodeReport>> = (0..4u32)
.map(|i| {
let b = sever_barrier.clone();
let d = done_barrier.clone();
NamespaceWorker::spawn(format!("severance-node{i}"), move || {
severance_worker_body(i, b, d)
})
})
.collect();
let fds: Vec<RawFd> = workers.iter().map(NamespaceWorker::fd).collect();
let (root, root_driver) = netns::root_handle_with_driver().expect("root rtnetlink handle");
let segments = vec![
Segment {
name: "slot0",
members: vec![
Member {
node: 0,
dev: "eth0",
},
Member {
node: 1,
dev: "eth0",
},
],
},
Segment {
name: "slot1",
members: vec![
Member {
node: 2,
dev: "eth0",
},
Member {
node: 3,
dev: "eth0",
},
],
},
];
let mut mesh = netns::wire(&root, &segments, &fds)
.await
.expect("wire severance segments");
mesh.link_bridges(&root, "slot0", "slot1", "sevup")
.await
.expect("uplink the two slots' bridges");
for w in &workers {
w.signal_moved();
}
let _ = tokio::time::timeout(
SEVERANCE_TIMEOUT + Duration::from_secs(5),
sever_barrier.wait(),
)
.await;
mesh.sever(&root, "sevup").await.expect("sever the uplink");
let _ = tokio::time::timeout(
SEVERANCE_TIMEOUT + Duration::from_secs(25),
done_barrier.wait(),
)
.await;
let results = netns::join_all(
workers,
SEVERANCE_TIMEOUT + Duration::from_secs(60) + netns::JOIN_MARGIN,
)
.await;
netns::teardown_mesh(mesh, root, root_driver).await;
for (i, r) in results.into_iter().enumerate() {
r.unwrap_or_else(|e| panic!("severance worker {i} failed: {e:?}"));
}
}
fn partition_position(label: &str) -> Vec<u32> {
match label {
"a" => vec![0, 0],
"b1" => vec![0, 1],
"b2" => vec![1, 1],
_ => unreachable!(),
}
}
enum PartitionReport {
A {
report: NodeReport,
saw_split_before_debounce: bool,
saw_split_after_debounce: bool,
},
B(NodeReport),
}
async fn partition_worker_a(
sever_barrier: Arc<Barrier>,
done_barrier: Arc<Barrier>,
) -> anyhow::Result<PartitionReport> {
let devs = vec!["eth0", "eth1"];
let dev_index = netns::bring_up_devs(&devs).await?;
let mut tasks = JoinSet::new();
let cancel = CancellationToken::new();
let started = netns::spawn_node(
NodeId::from_raw(501).unwrap(),
Some(partition_position("a")),
None,
&PARTITION_GSIZES,
&devs,
PARTITION_PORT,
&mut tasks,
cancel.clone(),
)
.await?;
let converged = wait_until(
|| {
let snapshot = started.running.generation.borrow().qspn.snapshot();
cost_at(&snapshot, 1, 1) == Some(Cost::Finite(netns::RTT_MS))
},
PARTITION_TIMEOUT,
)
.await;
anyhow::ensure!(converged, "a: did not converge to slot 1 before partition");
let mut events = started.running.generation.borrow().qspn.subscribe_events();
let _ = tokio::time::timeout(
PARTITION_TIMEOUT + Duration::from_secs(5),
sever_barrier.wait(),
)
.await;
let debounce_check_deadline = tokio::time::Instant::now() + Duration::from_secs(9);
let mut saw_split_before_debounce = false;
while tokio::time::Instant::now() < debounce_check_deadline {
if let Ok(Ok(ntk_qspn::QspnEvent::GnodeSplitted { .. })) =
tokio::time::timeout(Duration::from_millis(200), events.recv()).await
{
saw_split_before_debounce = true;
}
}
let mut saw_split_after_debounce = false;
let after_deadline = tokio::time::Instant::now() + Duration::from_secs(20);
while tokio::time::Instant::now() < after_deadline && !saw_split_after_debounce {
if let Ok(Ok(ntk_qspn::QspnEvent::GnodeSplitted { .. })) =
tokio::time::timeout(Duration::from_millis(200), events.recv()).await
{
saw_split_after_debounce = true;
}
}
let report = netns::observe("a", &started, dev_index).await?;
let _ = tokio::time::timeout(Duration::from_secs(30), done_barrier.wait()).await;
netns::teardown(&started, cancel, &mut tasks).await;
Ok(PartitionReport::A {
report,
saw_split_before_debounce,
saw_split_after_debounce,
})
}
async fn partition_worker_b(
label: &'static str,
my_id: NodeId,
other_pos0: u32,
sever_barrier: Arc<Barrier>,
done_barrier: Arc<Barrier>,
) -> anyhow::Result<PartitionReport> {
let devs = vec!["eth0", "eth1"];
let dev_index = netns::bring_up_devs(&devs).await?;
let mut tasks = JoinSet::new();
let cancel = CancellationToken::new();
let started = netns::spawn_node(
my_id,
Some(partition_position(label)),
None,
&PARTITION_GSIZES,
&devs,
PARTITION_PORT,
&mut tasks,
cancel.clone(),
)
.await?;
let converged = wait_until(
|| {
let snapshot = started.running.generation.borrow().qspn.snapshot();
cost_at(&snapshot, 0, other_pos0) == Some(Cost::Finite(netns::RTT_MS))
&& cost_at(&snapshot, 1, 0) == Some(Cost::Finite(netns::RTT_MS))
},
PARTITION_TIMEOUT,
)
.await;
anyhow::ensure!(
converged,
"{label}: did not converge to its sibling and to a's group before partition"
);
let _ = tokio::time::timeout(
PARTITION_TIMEOUT + Duration::from_secs(5),
sever_barrier.wait(),
)
.await;
let lost = wait_until(
|| {
let snapshot = started.running.generation.borrow().qspn.snapshot();
cost_at(&snapshot, 0, other_pos0).is_none()
&& cost_at(&snapshot, 1, 0) == Some(Cost::Finite(netns::RTT_MS))
},
Duration::from_secs(20),
)
.await;
anyhow::ensure!(
lost,
"{label}: did not drop exactly its severed-sibling route within 20s: {:#?}",
started.running.generation.borrow().qspn.snapshot()
);
let report = netns::observe(label, &started, dev_index).await?;
let _ = tokio::time::timeout(Duration::from_secs(30), done_barrier.wait()).await;
netns::teardown(&started, cancel, &mut tasks).await;
Ok(PartitionReport::B(report))
}
#[ignore = "requires the equivalent of CAP_NET_ADMIN over its own network namespaces"]
#[tokio::test]
async fn partition_signals_split_only_after_the_documented_debounce() {
init_tracing();
let sever_barrier = Arc::new(Barrier::new(4));
let done_barrier = Arc::new(Barrier::new(3));
let worker_a = NamespaceWorker::spawn("partition-a", {
let (sb, db) = (sever_barrier.clone(), done_barrier.clone());
move || partition_worker_a(sb, db)
});
let worker_b1 = NamespaceWorker::spawn("partition-b1", {
let (sb, db) = (sever_barrier.clone(), done_barrier.clone());
move || partition_worker_b("b1", NodeId::from_raw(502).unwrap(), 1, sb, db)
});
let worker_b2 = NamespaceWorker::spawn("partition-b2", {
let (sb, db) = (sever_barrier.clone(), done_barrier.clone());
move || partition_worker_b("b2", NodeId::from_raw(503).unwrap(), 0, sb, db)
});
let fd_a = worker_a.fd();
let fd_b1 = worker_b1.fd();
let fd_b2 = worker_b2.fd();
let (root, root_driver) = netns::root_handle_with_driver().expect("root rtnetlink handle");
let segments = vec![
Segment {
name: "ab1",
members: vec![
Member {
node: 0,
dev: "eth0",
},
Member {
node: 1,
dev: "eth0",
},
],
},
Segment {
name: "ab2",
members: vec![
Member {
node: 0,
dev: "eth1",
},
Member {
node: 2,
dev: "eth0",
},
],
},
Segment {
name: "b1b2",
members: vec![
Member {
node: 1,
dev: "eth1",
},
Member {
node: 2,
dev: "eth1",
},
],
},
];
let mesh = netns::wire(&root, &segments, &[fd_a, fd_b1, fd_b2])
.await
.expect("wire partition triangle");
worker_a.signal_moved();
worker_b1.signal_moved();
worker_b2.signal_moved();
let _ = tokio::time::timeout(
PARTITION_TIMEOUT + Duration::from_secs(5),
sever_barrier.wait(),
)
.await;
mesh.sever(&root, "b1b2").await.expect("sever b1b2 segment");
let mut results = netns::join_all(
vec![worker_a, worker_b1, worker_b2],
PARTITION_TIMEOUT + Duration::from_secs(60) + netns::JOIN_MARGIN,
)
.await;
netns::teardown_mesh(mesh, root, root_driver).await;
let report_b2 = results
.pop()
.unwrap()
.unwrap_or_else(|e| panic!("worker b2 failed: {e:?}"));
let report_b1 = results
.pop()
.unwrap()
.unwrap_or_else(|e| panic!("worker b1 failed: {e:?}"));
let report_a = results
.pop()
.unwrap()
.unwrap_or_else(|e| panic!("worker a failed: {e:?}"));
let PartitionReport::A {
report: report_a,
saw_split_before_debounce,
saw_split_after_debounce,
} = report_a
else {
panic!("worker a returned the wrong report variant");
};
assert!(
!saw_split_before_debounce,
"GnodeSplitted fired before the documented 10s debounce elapsed"
);
assert!(
saw_split_after_debounce,
"GnodeSplitted did not fire within 20s after the 10s debounce elapsed"
);
let topo = Topology::new(PARTITION_GSIZES).unwrap();
for (report, label, a_dev, other_pos0) in [
(report_b1, "b1", "eth0", 1u32),
(report_b2, "b2", "eth1", 0u32),
] {
let PartitionReport::B(report) = report else {
panic!("worker b returned the wrong report variant");
};
let naddr = Naddr::new(topo.clone(), partition_position(label)).unwrap();
let via = report_a
.linklocal(a_dev)
.unwrap_or_else(|| panic!("a has no linklocal address on {a_dev}"));
let expected = RouteSpec {
destination: addressing::gnode_destination(&naddr, HCoord::new(1, 0)).unwrap(),
table: report.route_table,
target: RouteTarget::Gateway {
via,
dev: Interface::Index(*report.dev_index.get("eth0").unwrap()),
src: Some(addressing::host_address(&naddr).unwrap().address()),
},
};
assert_eq!(
report.routes,
vec![expected],
"{label}: must have dropped exactly its severed route to node at pos0={other_pos0}, \
keeping only its route to a's group: {:#?}",
report.routes
);
}
}
const MERGE_GSIZES: [u32; 1] = [8];
const MERGE_PORT: u16 = 27340;
const GROUP_CONVERGE_TIMEOUT: Duration = Duration::from_secs(90);
const MERGE_TIMEOUT: Duration = Duration::from_secs(90);
const MERGE_CONFIG: MergeScenarioConfig = MergeScenarioConfig {
gsizes: &MERGE_GSIZES,
port: MERGE_PORT,
group_converge_timeout: GROUP_CONVERGE_TIMEOUT,
merge_timeout: MERGE_TIMEOUT,
};
#[derive(Debug)]
struct MergeReport {
report: NodeReport,
pre_network_id: i64,
pre_positions: Vec<u32>,
post_network_id: i64,
}
#[derive(Clone, Copy)]
struct MergeScenarioConfig {
gsizes: &'static [u32],
port: u16,
group_converge_timeout: Duration,
merge_timeout: Duration,
}
async fn merge_worker_body(
label: &'static str,
my_id: NodeId,
config: MergeScenarioConfig,
converge_barrier: Arc<Barrier>,
barrier: Arc<Barrier>,
) -> anyhow::Result<MergeReport> {
let devs = ["eth0"];
let dev_index = netns::bring_up_devs(&devs).await?;
let mut tasks = JoinSet::new();
let cancel = CancellationToken::new();
let started = netns::spawn_node(
my_id,
None,
None,
config.gsizes,
&devs,
config.port,
&mut tasks,
cancel.clone(),
)
.await?;
let group_converged = wait_until(
|| {
let snapshot = started.running.generation.borrow().qspn.snapshot();
snapshot
.levels
.first()
.is_some_and(|level| level.len() == 2)
},
config.group_converge_timeout,
)
.await;
anyhow::ensure!(
group_converged,
"{label}: did not converge to its own group's two siblings before the uplink: {:#?}",
started.running.generation.borrow().qspn.snapshot()
);
let pre_network_id = started.running.net.network_id();
let pre_positions = started
.running
.generation
.borrow()
.qspn
.my_naddr()
.positions()
.to_vec();
let _ = tokio::time::timeout(
config.group_converge_timeout + Duration::from_secs(5),
converge_barrier.wait(),
)
.await;
tokio::time::sleep(config.merge_timeout).await;
let report = netns::observe(label, &started, dev_index).await?;
let post_network_id = started.running.net.network_id();
let _ = tokio::time::timeout(Duration::from_secs(30), barrier.wait()).await;
netns::teardown(&started, cancel, &mut tasks).await;
Ok(MergeReport {
report,
pre_network_id,
pre_positions,
post_network_id,
})
}
#[ignore = "requires the equivalent of CAP_NET_ADMIN over its own network namespaces"]
#[tokio::test]
async fn two_star_groups_merge_into_one_network() {
init_tracing();
let converge_barrier = Arc::new(Barrier::new(7));
let barrier = Arc::new(Barrier::new(6));
let node = |idx: usize| -> (&'static str, NodeId) {
match idx {
0 => ("a0", NodeId::from_raw(601).unwrap()),
1 => ("a1", NodeId::from_raw(602).unwrap()),
2 => ("a2", NodeId::from_raw(603).unwrap()),
3 => ("b0", NodeId::from_raw(604).unwrap()),
4 => ("b1", NodeId::from_raw(605).unwrap()),
5 => ("b2", NodeId::from_raw(606).unwrap()),
_ => unreachable!(),
}
};
let workers: Vec<NamespaceWorker<MergeReport>> = (0..6)
.map(|i| {
let (label, id) = node(i);
let cb = converge_barrier.clone();
let b = barrier.clone();
NamespaceWorker::spawn(label, move || {
merge_worker_body(label, id, MERGE_CONFIG, cb, b)
})
})
.collect();
let fds: Vec<RawFd> = workers.iter().map(NamespaceWorker::fd).collect();
let (root, root_driver) = netns::root_handle_with_driver().expect("root rtnetlink handle");
let segments = vec![
Segment {
name: "grpa",
members: vec![
Member {
node: 0,
dev: "eth0",
},
Member {
node: 1,
dev: "eth0",
},
Member {
node: 2,
dev: "eth0",
},
],
},
Segment {
name: "grpb",
members: vec![
Member {
node: 3,
dev: "eth0",
},
Member {
node: 4,
dev: "eth0",
},
Member {
node: 5,
dev: "eth0",
},
],
},
];
let mut mesh = netns::wire(&root, &segments, &fds)
.await
.expect("wire merge topology");
for w in &workers {
w.signal_moved();
}
let _ = tokio::time::timeout(
GROUP_CONVERGE_TIMEOUT + Duration::from_secs(5),
converge_barrier.wait(),
)
.await;
mesh.link_bridges(&root, "grpa", "grpb", "abup")
.await
.expect("uplink the two groups' bridges");
let results = netns::join_all(
workers,
MERGE_TIMEOUT + GROUP_CONVERGE_TIMEOUT + netns::JOIN_MARGIN,
)
.await;
netns::teardown_mesh(mesh, root, root_driver).await;
let reports: Vec<MergeReport> = results
.into_iter()
.enumerate()
.map(|(i, r)| r.unwrap_or_else(|e| panic!("merge worker {i} failed: {e:?}")))
.collect();
for r in &reports {
eprintln!(
"{}: rehooked={} pre_network_id={} pre_positions={:?} post_network_id={} \
post_positions={:?} routes={:#?}",
r.report.label,
r.report.rehooked,
r.pre_network_id,
r.pre_positions,
r.post_network_id,
r.report.naddr_positions,
r.report.routes
);
}
let post_network_ids: std::collections::HashSet<i64> =
reports.iter().map(|r| r.post_network_id).collect();
assert_eq!(
post_network_ids.len(),
1,
"all six nodes must share one network_id after the merge: {:?}",
reports
.iter()
.map(|r| (r.report.label.clone(), r.post_network_id))
.collect::<Vec<_>>()
);
let post_positions: std::collections::HashSet<Vec<u32>> = reports
.iter()
.map(|r| r.report.naddr_positions.clone())
.collect();
assert_eq!(
post_positions.len(),
reports.len(),
"all six nodes must land at distinct positions in the merged network: {:?}",
reports
.iter()
.map(|r| (&r.report.label, &r.report.naddr_positions))
.collect::<Vec<_>>()
);
for r in reports
.iter()
.filter(|r| r.pre_positions != r.report.naddr_positions)
{
assert!(
!r.report.routes.is_empty(),
"{} moved but has no real kernel routes after the merge: {:#?}",
r.report.label,
r.report.routes
);
}
}
const UNIT_MERGE_GSIZES: [u32; 2] = [4, 2];
const UNIT_MERGE_PORT: u16 = 27350;
const UNIT_GROUP_CONVERGE_TIMEOUT: Duration = Duration::from_secs(180);
const UNIT_MERGE_TIMEOUT: Duration = Duration::from_secs(180);
const UNIT_MERGE_CONFIG: MergeScenarioConfig = MergeScenarioConfig {
gsizes: &UNIT_MERGE_GSIZES,
port: UNIT_MERGE_PORT,
group_converge_timeout: UNIT_GROUP_CONVERGE_TIMEOUT,
merge_timeout: UNIT_MERGE_TIMEOUT,
};
#[ignore = "requires the equivalent of CAP_NET_ADMIN over its own network namespaces"]
#[tokio::test]
async fn two_level_gnode_migrates_as_a_unit_into_merged_network() {
init_tracing();
let converge_barrier = Arc::new(Barrier::new(7));
let barrier = Arc::new(Barrier::new(6));
let node = |idx: usize| -> (&'static str, NodeId) {
match idx {
0 => ("a0", NodeId::from_raw(701).unwrap()),
1 => ("a1", NodeId::from_raw(702).unwrap()),
2 => ("a2", NodeId::from_raw(703).unwrap()),
3 => ("b0", NodeId::from_raw(704).unwrap()),
4 => ("b1", NodeId::from_raw(705).unwrap()),
5 => ("b2", NodeId::from_raw(706).unwrap()),
_ => unreachable!(),
}
};
let workers: Vec<NamespaceWorker<MergeReport>> = (0..6)
.map(|i| {
let (label, id) = node(i);
let cb = converge_barrier.clone();
let b = barrier.clone();
NamespaceWorker::spawn(label, move || {
merge_worker_body(label, id, UNIT_MERGE_CONFIG, cb, b)
})
})
.collect();
let fds: Vec<RawFd> = workers.iter().map(NamespaceWorker::fd).collect();
let (root, root_driver) = netns::root_handle_with_driver().expect("root rtnetlink handle");
let segments = vec![
Segment {
name: "ugrpa",
members: vec![
Member {
node: 0,
dev: "eth0",
},
Member {
node: 1,
dev: "eth0",
},
Member {
node: 2,
dev: "eth0",
},
],
},
Segment {
name: "ugrpb",
members: vec![
Member {
node: 3,
dev: "eth0",
},
Member {
node: 4,
dev: "eth0",
},
Member {
node: 5,
dev: "eth0",
},
],
},
];
let mut mesh = netns::wire(&root, &segments, &fds)
.await
.expect("wire unit-migration topology");
for w in &workers {
w.signal_moved();
}
let _ = tokio::time::timeout(
UNIT_GROUP_CONVERGE_TIMEOUT + Duration::from_secs(5),
converge_barrier.wait(),
)
.await;
mesh.link_bridges(&root, "ugrpa", "ugrpb", "uabup")
.await
.expect("uplink the two g-nodes' bridges");
let results = netns::join_all(
workers,
UNIT_MERGE_TIMEOUT + UNIT_GROUP_CONVERGE_TIMEOUT + netns::JOIN_MARGIN,
)
.await;
netns::teardown_mesh(mesh, root, root_driver).await;
let reports: Vec<MergeReport> = results
.into_iter()
.enumerate()
.map(|(i, r)| r.unwrap_or_else(|e| panic!("unit-migration worker {i} failed: {e:?}")))
.collect();
for r in &reports {
eprintln!(
"{}: pre_network_id={} pre_positions={:?} post_network_id={} post_positions={:?} \
routes={:#?}",
r.report.label,
r.pre_network_id,
r.pre_positions,
r.post_network_id,
r.report.naddr_positions,
r.report.routes
);
}
let a_labels = ["a0", "a1", "a2"];
let b_labels = ["b0", "b1", "b2"];
for labels in [&a_labels[..], &b_labels[..]] {
let level1: std::collections::HashSet<u32> = reports
.iter()
.filter(|r| labels.contains(&r.report.label.as_str()))
.map(|r| r.pre_positions[1])
.collect();
assert_eq!(
level1.len(),
1,
"{labels:?} must share one level-1 g-node before the uplink: {:?}",
reports
.iter()
.filter(|r| labels.contains(&r.report.label.as_str()))
.map(|r| (&r.report.label, &r.pre_positions))
.collect::<Vec<_>>()
);
}
let post_network_ids: std::collections::HashSet<i64> =
reports.iter().map(|r| r.post_network_id).collect();
assert_eq!(
post_network_ids.len(),
1,
"all six nodes must share one network_id after the merge: {:?}",
reports
.iter()
.map(|r| (r.report.label.clone(), r.post_network_id))
.collect::<Vec<_>>()
);
let moved = |labels: &[&str]| -> usize {
reports
.iter()
.filter(|r| {
labels.contains(&r.report.label.as_str())
&& r.pre_positions != r.report.naddr_positions
})
.count()
};
let a_moved = moved(&a_labels);
let b_moved = moved(&b_labels);
assert!(
(a_moved == 0) != (b_moved == 0),
"exactly one g-node's members should migrate for the merge, never both or neither: \
a_moved={a_moved} b_moved={b_moved}"
);
let (losing_labels, winning_labels): (&[&str], &[&str]) = if a_moved > 0 {
(&a_labels, &b_labels)
} else {
(&b_labels, &a_labels)
};
assert_eq!(
moved(losing_labels),
losing_labels.len(),
"the losing g-node must migrate as a whole unit, not partially: {:?}",
reports
.iter()
.filter(|r| losing_labels.contains(&r.report.label.as_str()))
.map(|r| (&r.report.label, &r.pre_positions, &r.report.naddr_positions))
.collect::<Vec<_>>()
);
assert_eq!(
moved(winning_labels),
0,
"the winning g-node must keep its own positions untouched by the merge: {:?}",
reports
.iter()
.filter(|r| winning_labels.contains(&r.report.label.as_str()))
.map(|r| (&r.report.label, &r.pre_positions, &r.report.naddr_positions))
.collect::<Vec<_>>()
);
let winner_level1: std::collections::HashSet<u32> = reports
.iter()
.filter(|r| winning_labels.contains(&r.report.label.as_str()))
.map(|r| r.report.naddr_positions[1])
.collect();
assert_eq!(
winner_level1.len(),
1,
"the winning g-node must keep one shared level-1 position: {:?}",
reports
.iter()
.filter(|r| winning_labels.contains(&r.report.label.as_str()))
.map(|r| (&r.report.label, &r.report.naddr_positions))
.collect::<Vec<_>>()
);
let winner_level1 = *winner_level1
.iter()
.next()
.expect("checked len() == 1 above");
for label in losing_labels {
let report = reports
.iter()
.find(|r| r.report.label == *label)
.unwrap_or_else(|| panic!("missing report for {label}"));
assert_eq!(
report.report.naddr_positions[1], winner_level1,
"{label} (losing g-node) must share the winner's level-1 position after migrating \
as a unit: {:?}",
report.report.naddr_positions
);
}
let post_positions: std::collections::HashSet<Vec<u32>> = reports
.iter()
.map(|r| r.report.naddr_positions.clone())
.collect();
assert_eq!(
post_positions.len(),
reports.len(),
"all six nodes must land at distinct positions in the merged network: {:?}",
reports
.iter()
.map(|r| (&r.report.label, &r.report.naddr_positions))
.collect::<Vec<_>>()
);
for label in losing_labels {
let report = reports
.iter()
.find(|r| r.report.label == *label)
.unwrap_or_else(|| panic!("missing report for {label}"));
assert!(
!report.report.routes.is_empty(),
"{label} (losing g-node) has no real kernel routes after the merge: {:#?}",
report.report.routes
);
}
}
const ISOLATED_MERGE_PORT: u16 = 27360;
const ISOLATED_GROUP_CONVERGE_TIMEOUT: Duration = Duration::from_secs(20);
const ISOLATED_MERGE_TIMEOUT: Duration = Duration::from_secs(140);
const ISOLATED_MERGE_CONFIG: MergeScenarioConfig = MergeScenarioConfig {
gsizes: &UNIT_MERGE_GSIZES,
port: ISOLATED_MERGE_PORT,
group_converge_timeout: ISOLATED_GROUP_CONVERGE_TIMEOUT,
merge_timeout: ISOLATED_MERGE_TIMEOUT,
};
fn isolated_position(idx: u32) -> Vec<u32> {
match idx {
0 => vec![0, 0],
1 => vec![1, 0],
2 => vec![2, 0],
3 => vec![0, 1],
4 => vec![1, 1],
5 => vec![2, 1],
_ => unreachable!("isolated-merge mesh has exactly 6 nodes"),
}
}
fn isolated_network_id(idx: u32) -> i64 {
match idx {
0..=2 => 91_000_001,
3..=5 => 91_000_002,
_ => unreachable!("isolated-merge mesh has exactly 6 nodes"),
}
}
fn isolated_preformed(idx: u32) -> PreformedNetwork {
PreformedNetwork {
network_id: isolated_network_id(idx),
position: isolated_position(idx),
}
}
async fn isolated_merge_worker_body(
label: &'static str,
my_id: NodeId,
preformed: PreformedNetwork,
config: MergeScenarioConfig,
converge_barrier: Arc<Barrier>,
barrier: Arc<Barrier>,
) -> anyhow::Result<MergeReport> {
let devs = ["eth0"];
let dev_index = netns::bring_up_devs(&devs).await?;
let mut tasks = JoinSet::new();
let cancel = CancellationToken::new();
let started = netns::spawn_node(
my_id,
None,
Some(preformed),
config.gsizes,
&devs,
config.port,
&mut tasks,
cancel.clone(),
)
.await?;
let group_converged = wait_until(
|| {
let snapshot = started.running.generation.borrow().qspn.snapshot();
snapshot
.levels
.first()
.is_some_and(|level| level.len() == 2)
},
config.group_converge_timeout,
)
.await;
anyhow::ensure!(
group_converged,
"{label}: did not converge to its own group's two siblings before the uplink: {:#?}",
started.running.generation.borrow().qspn.snapshot()
);
let pre_network_id = started.running.net.network_id();
let pre_positions = started
.running
.generation
.borrow()
.qspn
.my_naddr()
.positions()
.to_vec();
let _ = tokio::time::timeout(
config.group_converge_timeout + Duration::from_secs(5),
converge_barrier.wait(),
)
.await;
tokio::time::sleep(config.merge_timeout).await;
let report = netns::observe(label, &started, dev_index).await?;
let post_network_id = started.running.net.network_id();
let _ = tokio::time::timeout(Duration::from_secs(30), barrier.wait()).await;
netns::teardown(&started, cancel, &mut tasks).await;
Ok(MergeReport {
report,
pre_network_id,
pre_positions,
post_network_id,
})
}
#[ignore = "requires the equivalent of CAP_NET_ADMIN over its own network namespaces"]
#[tokio::test]
async fn isolated_merge_migrates_a_preformed_losing_gnode_as_a_unit() {
init_tracing();
let converge_barrier = Arc::new(Barrier::new(7));
let barrier = Arc::new(Barrier::new(6));
let node = |idx: usize| -> (&'static str, NodeId) {
match idx {
0 => ("a0", NodeId::from_raw(801).unwrap()),
1 => ("a1", NodeId::from_raw(802).unwrap()),
2 => ("a2", NodeId::from_raw(803).unwrap()),
3 => ("b0", NodeId::from_raw(804).unwrap()),
4 => ("b1", NodeId::from_raw(805).unwrap()),
5 => ("b2", NodeId::from_raw(806).unwrap()),
_ => unreachable!(),
}
};
let workers: Vec<NamespaceWorker<MergeReport>> = (0..6)
.map(|i| {
let (label, id) = node(i);
let cb = converge_barrier.clone();
let b = barrier.clone();
NamespaceWorker::spawn(label, move || {
isolated_merge_worker_body(
label,
id,
isolated_preformed(i as u32),
ISOLATED_MERGE_CONFIG,
cb,
b,
)
})
})
.collect();
let fds: Vec<RawFd> = workers.iter().map(NamespaceWorker::fd).collect();
let (root, root_driver) = netns::root_handle_with_driver().expect("root rtnetlink handle");
let segments = vec![
Segment {
name: "igrpa",
members: vec![
Member {
node: 0,
dev: "eth0",
},
Member {
node: 1,
dev: "eth0",
},
Member {
node: 2,
dev: "eth0",
},
],
},
Segment {
name: "igrpb",
members: vec![
Member {
node: 3,
dev: "eth0",
},
Member {
node: 4,
dev: "eth0",
},
Member {
node: 5,
dev: "eth0",
},
],
},
];
let mut mesh = netns::wire(&root, &segments, &fds)
.await
.expect("wire isolated-merge topology");
for w in &workers {
w.signal_moved();
}
let _ = tokio::time::timeout(
ISOLATED_GROUP_CONVERGE_TIMEOUT + Duration::from_secs(5),
converge_barrier.wait(),
)
.await;
mesh.link_bridges(&root, "igrpa", "igrpb", "iabup")
.await
.expect("uplink the two g-nodes' bridges");
let results = netns::join_all(
workers,
ISOLATED_MERGE_TIMEOUT + ISOLATED_GROUP_CONVERGE_TIMEOUT + netns::JOIN_MARGIN,
)
.await;
netns::teardown_mesh(mesh, root, root_driver).await;
let reports: Vec<MergeReport> = results
.into_iter()
.enumerate()
.map(|(i, r)| r.unwrap_or_else(|e| panic!("isolated-merge worker {i} failed: {e:?}")))
.collect();
for r in &reports {
eprintln!(
"{}: pre_network_id={} pre_positions={:?} post_network_id={} post_positions={:?} \
routes={:#?}",
r.report.label,
r.pre_network_id,
r.pre_positions,
r.post_network_id,
r.report.naddr_positions,
r.report.routes
);
}
let a_labels = ["a0", "a1", "a2"];
let b_labels = ["b0", "b1", "b2"];
for labels in [&a_labels[..], &b_labels[..]] {
let level1: std::collections::HashSet<u32> = reports
.iter()
.filter(|r| labels.contains(&r.report.label.as_str()))
.map(|r| r.pre_positions[1])
.collect();
assert_eq!(
level1.len(),
1,
"{labels:?} must share one level-1 g-node before the uplink: {:?}",
reports
.iter()
.filter(|r| labels.contains(&r.report.label.as_str()))
.map(|r| (&r.report.label, &r.pre_positions))
.collect::<Vec<_>>()
);
let pre_network_ids: std::collections::HashSet<i64> = reports
.iter()
.filter(|r| labels.contains(&r.report.label.as_str()))
.map(|r| r.pre_network_id)
.collect();
assert_eq!(
pre_network_ids.len(),
1,
"{labels:?} must already share one network_id before the uplink — the piece a bare \
`initial_position` could never establish: {:?}",
reports
.iter()
.filter(|r| labels.contains(&r.report.label.as_str()))
.map(|r| (&r.report.label, r.pre_network_id))
.collect::<Vec<_>>()
);
}
let post_network_ids: std::collections::HashSet<i64> =
reports.iter().map(|r| r.post_network_id).collect();
assert_eq!(
post_network_ids.len(),
1,
"all six nodes must share one network_id after the merge: {:?}",
reports
.iter()
.map(|r| (r.report.label.clone(), r.post_network_id))
.collect::<Vec<_>>()
);
let adopted = |labels: &[&str]| -> usize {
reports
.iter()
.filter(|r| {
labels.contains(&r.report.label.as_str()) && r.pre_network_id != r.post_network_id
})
.count()
};
let a_adopted = adopted(&a_labels);
let b_adopted = adopted(&b_labels);
assert!(
(a_adopted == 0) != (b_adopted == 0),
"exactly one g-node must adopt the other's network_id, never both or neither: \
a_adopted={a_adopted} b_adopted={b_adopted}"
);
let (losing_labels, winning_labels): (&[&str], &[&str]) = if a_adopted > 0 {
(&a_labels, &b_labels)
} else {
(&b_labels, &a_labels)
};
assert_eq!(
adopted(losing_labels),
losing_labels.len(),
"the losing g-node must migrate as a whole unit, not partially — every member adopts the \
winner's network_id: {:?}",
reports
.iter()
.filter(|r| losing_labels.contains(&r.report.label.as_str()))
.map(|r| (
&r.report.label,
r.pre_network_id,
r.post_network_id,
&r.pre_positions,
&r.report.naddr_positions
))
.collect::<Vec<_>>()
);
assert_eq!(
adopted(winning_labels),
0,
"the winning g-node must keep its own network_id untouched by the merge: {:?}",
reports
.iter()
.filter(|r| winning_labels.contains(&r.report.label.as_str()))
.map(|r| (&r.report.label, r.pre_network_id, r.post_network_id))
.collect::<Vec<_>>()
);
let shared_level1 = |labels: &[&str]| -> u32 {
let level1: std::collections::HashSet<u32> = reports
.iter()
.filter(|r| labels.contains(&r.report.label.as_str()))
.map(|r| r.report.naddr_positions[1])
.collect();
assert_eq!(
level1.len(),
1,
"{labels:?} must share exactly one level-1 position after the merge: {:?}",
reports
.iter()
.filter(|r| labels.contains(&r.report.label.as_str()))
.map(|r| (&r.report.label, &r.report.naddr_positions))
.collect::<Vec<_>>()
);
*level1.iter().next().expect("checked len() == 1 above")
};
let winner_level1 = shared_level1(winning_labels);
let loser_level1 = shared_level1(losing_labels);
assert_ne!(
loser_level1,
winner_level1,
"the migrated g-node must occupy its own level-1 slot, not the winner's: {:?}",
reports
.iter()
.map(|r| (&r.report.label, &r.report.naddr_positions))
.collect::<Vec<_>>()
);
for report in &reports {
assert_eq!(
report.report.naddr_positions[0], report.pre_positions[0],
"{} must keep its level-0 position through a unit migration: {:?} -> {:?}",
report.report.label, report.pre_positions, report.report.naddr_positions
);
}
let post_positions: std::collections::HashSet<Vec<u32>> = reports
.iter()
.map(|r| r.report.naddr_positions.clone())
.collect();
assert_eq!(
post_positions.len(),
reports.len(),
"all six nodes must land at distinct positions in the merged network: {:?}",
reports
.iter()
.map(|r| (&r.report.label, &r.report.naddr_positions))
.collect::<Vec<_>>()
);
for label in losing_labels {
let report = reports
.iter()
.find(|r| r.report.label == *label)
.unwrap_or_else(|| panic!("missing report for {label}"));
assert!(
!report.report.routes.is_empty(),
"{label} (losing g-node) has no real kernel routes after the merge: {:#?}",
report.report.routes
);
}
}