use std::net::Ipv4Addr;
use std::os::fd::OwnedFd;
use std::time::{Duration, Instant};
use nix::fcntl::{FcntlArg, OFlag, fcntl};
use nix::sys::wait::{WaitStatus, waitpid};
use nix::unistd::{ForkResult, Pid, fork, pipe, read, write};
use nlink::netlink::link::VethLink;
use nlink::netlink::{Connection, Route};
use crate::Error;
use crate::netns::{enter_sibling_net_namespace, enter_unprivileged_net_namespace};
const SUBNET_PREFIX: u8 = 24;
const ADDR_A: Ipv4Addr = Ipv4Addr::new(10, 200, 100, 1);
const ADDR_B: Ipv4Addr = Ipv4Addr::new(10, 200, 100, 2);
const IFACE_A: &str = "veth-a";
const IFACE_B: &str = "veth-b";
#[derive(Debug)]
pub struct VethEnd {
pub interface: &'static str,
pub address: Ipv4Addr,
pub peer_address: Ipv4Addr,
my_done_w: OwnedFd,
peer_done_r: OwnedFd,
}
impl VethEnd {
pub fn signal_done(&self) -> Result<(), Error> {
write(&self.my_done_w, &[1u8]).map_err(Error::Pipe)?;
Ok(())
}
pub fn wait_for_peer(&self, timeout: Duration) -> Result<bool, Error> {
let deadline = Instant::now() + timeout;
let mut buf = [0u8; 1];
loop {
match read(&self.peer_done_r, &mut buf) {
Ok(1) => return Ok(true),
Ok(_) => return Ok(false),
Err(nix::Error::EAGAIN) => {
if Instant::now() >= deadline {
return Ok(false);
}
std::thread::sleep(Duration::from_millis(20));
}
Err(e) => return Err(Error::Pipe(e)),
}
}
}
}
fn set_nonblocking(fd: &OwnedFd) -> Result<(), Error> {
let flags = fcntl(fd, FcntlArg::F_GETFL).map_err(Error::Pipe)?;
let mut oflags = OFlag::from_bits_truncate(flags);
oflags.insert(OFlag::O_NONBLOCK);
fcntl(fd, FcntlArg::F_SETFL(oflags)).map_err(Error::Pipe)?;
Ok(())
}
struct SideAPipes {
b_ready_r: OwnedFd,
wire_done_w: OwnedFd,
b_end_up_r: OwnedFd,
b_result_w: OwnedFd,
a_done_w: OwnedFd,
b_done_r: OwnedFd,
}
struct SideBPipes {
b_ready_w: OwnedFd,
wire_done_r: OwnedFd,
b_end_up_w: OwnedFd,
b_done_w: OwnedFd,
a_done_r: OwnedFd,
}
pub fn fork_veth_pair<FA, FB>(a_fn: FA, b_fn: FB) -> Result<(i32, i32), Error>
where
FA: FnOnce(VethEnd) -> i32,
FB: FnOnce(VethEnd) -> i32,
{
let (b_ready_r, b_ready_w) = pipe().map_err(Error::Pipe)?;
let (wire_done_r, wire_done_w) = pipe().map_err(Error::Pipe)?;
let (b_end_up_r, b_end_up_w) = pipe().map_err(Error::Pipe)?;
let (b_result_r, b_result_w) = pipe().map_err(Error::Pipe)?;
let (a_done_r, a_done_w) = pipe().map_err(Error::Pipe)?;
let (b_done_r, b_done_w) = pipe().map_err(Error::Pipe)?;
match unsafe { fork() }.map_err(Error::Fork)? {
ForkResult::Parent { child: a_pid } => {
drop(b_ready_r);
drop(b_ready_w);
drop(wire_done_r);
drop(wire_done_w);
drop(b_end_up_r);
drop(b_end_up_w);
drop(b_result_w);
drop(a_done_r);
drop(a_done_w);
drop(b_done_r);
drop(b_done_w);
let a_code = match waitpid(a_pid, None).map_err(Error::Wait)? {
WaitStatus::Exited(_, code) => code,
other => return Err(Error::ChildTerminated(other)),
};
let mut buf = [0u8; 1];
let b_code = match read(&b_result_r, &mut buf) {
Ok(1) => i32::from(buf[0]),
_ => -1,
};
Ok((a_code, b_code))
}
ForkResult::Child => {
drop(b_result_r);
let code = run_side_a(
SideAPipes {
b_ready_r,
wire_done_w,
b_end_up_r,
b_result_w,
a_done_w,
b_done_r,
},
SideBPipes {
b_ready_w,
wire_done_r,
b_end_up_w,
b_done_w,
a_done_r,
},
a_fn,
b_fn,
);
std::process::exit(code);
}
}
}
fn run_side_a<FA, FB>(a_pipes: SideAPipes, b_pipes: SideBPipes, a_fn: FA, b_fn: FB) -> i32
where
FA: FnOnce(VethEnd) -> i32,
FB: FnOnce(VethEnd) -> i32,
{
if enter_unprivileged_net_namespace().is_err() {
return 121;
}
let b_pid = match unsafe { fork() } {
Ok(ForkResult::Child) => {
drop(a_pipes.wire_done_w);
drop(a_pipes.b_end_up_r);
drop(a_pipes.b_result_w);
drop(a_pipes.a_done_w);
drop(a_pipes.b_done_r);
let code = run_side_b(a_pipes.b_ready_r, b_pipes, b_fn);
std::process::exit(code);
}
Ok(ForkResult::Parent { child }) => child,
Err(_) => return 122,
};
drop(b_pipes.b_ready_w);
drop(b_pipes.wire_done_r);
drop(b_pipes.b_end_up_w);
drop(b_pipes.b_done_w);
drop(b_pipes.a_done_r);
let mut ready_buf = [0u8; 1];
match read(&a_pipes.b_ready_r, &mut ready_buf) {
Ok(1) => {}
_ => return 123,
}
let end_a = match configure_side_a(b_pid, a_pipes.a_done_w, a_pipes.b_done_r) {
Ok(end) => end,
Err(_) => return 124,
};
if write(&a_pipes.wire_done_w, &[1u8]).is_err() {
return 125;
}
let mut end_up_buf = [0u8; 1];
match read(&a_pipes.b_end_up_r, &mut end_up_buf) {
Ok(1) => {}
_ => return 128,
}
let a_result = a_fn(end_a);
let b_exit = match waitpid(b_pid, None) {
Ok(WaitStatus::Exited(_, code)) => code,
_ => return 126,
};
#[allow(clippy::cast_sign_loss, clippy::cast_possible_truncation)]
let b_byte = b_exit.clamp(0, 255) as u8;
if write(&a_pipes.b_result_w, &[b_byte]).is_err() {
return 127;
}
a_result
}
fn run_side_b<FB>(a_b_ready_r: OwnedFd, pipes: SideBPipes, b_fn: FB) -> i32
where
FB: FnOnce(VethEnd) -> i32,
{
drop(a_b_ready_r);
if enter_sibling_net_namespace().is_err() {
return 131;
}
if write(&pipes.b_ready_w, &[1u8]).is_err() {
return 132;
}
let mut done_buf = [0u8; 1];
match read(&pipes.wire_done_r, &mut done_buf) {
Ok(1) => {}
_ => return 133,
}
let end_b = match configure_side_b(pipes.b_done_w, pipes.a_done_r) {
Ok(end) => end,
Err(_) => return 134,
};
if write(&pipes.b_end_up_w, &[1u8]).is_err() {
return 135;
}
b_fn(end_b)
}
fn configure_side_a(
peer_pid: Pid,
my_done_w: OwnedFd,
peer_done_r: OwnedFd,
) -> Result<VethEnd, Error> {
set_nonblocking(&peer_done_r)?;
let runtime = tokio::runtime::Runtime::new().map_err(Error::Runtime)?;
runtime.block_on(async {
let conn = Connection::<Route>::new().map_err(Error::Netlink)?;
#[allow(clippy::cast_sign_loss)]
let peer_pid_raw = peer_pid.as_raw() as u32;
let veth = VethLink::new(IFACE_A, IFACE_B).peer_netns_pid(peer_pid_raw);
conn.add_link(veth).await.map_err(Error::Netlink)?;
conn.add_address(nlink::netlink::addr::Ipv4Address::new(
IFACE_A,
ADDR_A,
SUBNET_PREFIX,
))
.await
.map_err(Error::Netlink)?;
conn.set_link_up(IFACE_A).await.map_err(Error::Netlink)?;
conn.set_link_up("lo").await.map_err(Error::Netlink)?;
Ok(VethEnd {
interface: IFACE_A,
address: ADDR_A,
peer_address: ADDR_B,
my_done_w,
peer_done_r,
})
})
}
fn configure_side_b(my_done_w: OwnedFd, peer_done_r: OwnedFd) -> Result<VethEnd, Error> {
set_nonblocking(&peer_done_r)?;
let runtime = tokio::runtime::Runtime::new().map_err(Error::Runtime)?;
runtime.block_on(async {
let conn = Connection::<Route>::new().map_err(Error::Netlink)?;
conn.add_address(nlink::netlink::addr::Ipv4Address::new(
IFACE_B,
ADDR_B,
SUBNET_PREFIX,
))
.await
.map_err(Error::Netlink)?;
conn.set_link_up(IFACE_B).await.map_err(Error::Netlink)?;
conn.set_link_up("lo").await.map_err(Error::Netlink)?;
Ok(VethEnd {
interface: IFACE_B,
address: ADDR_B,
peer_address: ADDR_A,
my_done_w,
peer_done_r,
})
})
}
#[cfg(test)]
mod tests {
use std::net::UdpSocket;
use std::time::{Duration, Instant};
use super::*;
const PORT: u16 = 9999;
#[test]
fn fork_veth_pair_carries_real_udp_traffic() {
let (a_code, b_code) = fork_veth_pair(
|end: VethEnd| {
let socket = match UdpSocket::bind((end.address, 0)) {
Ok(s) => s,
Err(_) => return 2,
};
if socket
.set_read_timeout(Some(Duration::from_millis(200)))
.is_err()
{
return 3;
}
let peer = (end.peer_address, PORT);
let mut buf = [0u8; 4];
for _ in 0..10 {
if socket.send_to(b"ping", peer).is_err() {
std::thread::sleep(Duration::from_millis(20));
continue;
}
match socket.recv_from(&mut buf) {
Ok((n, from)) if &buf[..n] == b"pong" && from.ip() == end.peer_address => {
return 0;
}
_ => continue,
}
}
5
},
|end: VethEnd| {
let socket = match UdpSocket::bind((end.address, PORT)) {
Ok(s) => s,
Err(_) => return 2,
};
if socket
.set_read_timeout(Some(Duration::from_secs(6)))
.is_err()
{
return 3;
}
let mut buf = [0u8; 4];
loop {
match socket.recv_from(&mut buf) {
Ok((n, from)) if &buf[..n] == b"ping" => {
return if socket.send_to(b"pong", from).is_ok() {
0
} else {
4
};
}
Ok(_) => continue,
Err(e) => {
eprintln!("gateflow test: B never received a ping: {e}");
return 5;
}
}
}
},
)
.expect("fork_veth_pair should run to completion");
eprintln!("gateflow test: a_code={a_code} b_code={b_code}");
assert_eq!(
a_code, 0,
"side A did not complete the ping/pong (code {a_code})"
);
assert_eq!(
b_code, 0,
"side B did not complete the ping/pong (code {b_code})"
);
}
#[test]
fn veth_end_signal_done_and_wait_for_peer_synchronizes() {
let (a_code, b_code) = fork_veth_pair(
|end: VethEnd| {
std::thread::sleep(Duration::from_millis(300));
if end.signal_done().is_err() {
return 2;
}
0
},
|end: VethEnd| {
let start = Instant::now();
let signaled = match end.wait_for_peer(Duration::from_secs(2)) {
Ok(s) => s,
Err(_) => return 3,
};
let elapsed = start.elapsed();
if !signaled {
return 4;
}
if elapsed < Duration::from_millis(200) {
eprintln!(
"gateflow test: wait_for_peer returned after only {elapsed:?}, expected close to 300ms"
);
return 5;
}
0
},
)
.expect("fork_veth_pair should run to completion");
assert_eq!(a_code, 0, "side A failed (code {a_code})");
assert_eq!(b_code, 0, "side B failed (code {b_code})");
}
#[test]
fn veth_end_wait_for_peer_times_out_if_peer_never_signals() {
let (a_code, b_code) = fork_veth_pair(
|_end: VethEnd| {
std::thread::sleep(Duration::from_secs(1));
0
},
|end: VethEnd| {
let start = Instant::now();
let signaled = match end.wait_for_peer(Duration::from_millis(300)) {
Ok(s) => s,
Err(_) => return 2,
};
let elapsed = start.elapsed();
if signaled {
return 3;
}
if elapsed < Duration::from_millis(250) || elapsed > Duration::from_millis(700) {
eprintln!(
"gateflow test: wait_for_peer took {elapsed:?}, expected close to 300ms"
);
return 4;
}
0
},
)
.expect("fork_veth_pair should run to completion");
assert_eq!(a_code, 0, "side A failed (code {a_code})");
assert_eq!(b_code, 0, "side B failed (code {b_code})");
}
}