use ark_ff::Field;
use async_trait::async_trait;
use quinn::{ClientConfig, Connection, Endpoint, IdleTimeout, ServerConfig, TransportConfig};
use rustls::pki_types::{CertificateDer, PrivateKeyDer, PrivatePkcs8KeyDer};
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::future::Future;
use std::net::SocketAddr;
use std::pin::Pin;
use std::sync::Arc;
use stoffelnet::network_utils::{
ClientId, Message, Network, NetworkError, Node, PartyId, VerifiedOrdering,
};
use tokio::sync::Mutex;
use tracing::debug;
use uuid::Uuid;
type PeerFuture<'a, T> = Pin<Box<dyn Future<Output = Result<T, String>> + Send + 'a>>;
type BoxedPeerConnection = Box<dyn PeerConnection>;
type SharedPeerConnection = Arc<Mutex<BoxedPeerConnection>>;
type PeerConnectionMap = Arc<Mutex<HashMap<PartyId, SharedPeerConnection>>>;
const DEFAULT_IDLE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30);
const DEFAULT_KEEP_ALIVE_INTERVAL: std::time::Duration = std::time::Duration::from_secs(5);
fn client_endpoint_bind_addr() -> SocketAddr {
SocketAddr::from(([0, 0, 0, 0], 0))
}
fn party_id_from_uuid(uuid: Uuid) -> PartyId {
let mut id = PartyId::default();
for (index, byte) in uuid.as_bytes().iter().enumerate() {
let shift = (index % std::mem::size_of::<PartyId>()) * 8;
id ^= PartyId::from(*byte) << shift;
}
id
}
fn random_party_id() -> PartyId {
party_id_from_uuid(Uuid::new_v4())
}
fn party_id_to_u128(id: PartyId) -> u128 {
let mut bytes = [0u8; 16];
let id_bytes = id.to_le_bytes();
bytes[..id_bytes.len()].copy_from_slice(&id_bytes);
u128::from_le_bytes(bytes)
}
pub trait PeerConnection: Send + Sync {
fn send<'a>(&'a mut self, data: &'a [u8]) -> PeerFuture<'a, ()>;
fn receive<'a>(&'a mut self) -> PeerFuture<'a, Vec<u8>>;
fn send_on_stream<'a>(&'a mut self, stream_id: u64, data: &'a [u8]) -> PeerFuture<'a, ()>;
fn receive_from_stream<'a>(&'a mut self, stream_id: u64) -> PeerFuture<'a, Vec<u8>>;
fn remote_address(&self) -> SocketAddr;
fn close<'a>(&'a mut self) -> PeerFuture<'a, ()>;
}
pub trait NetworkManager: Send + Sync {
fn connect<'a>(&'a mut self, address: SocketAddr) -> PeerFuture<'a, BoxedPeerConnection>;
fn accept<'a>(&'a mut self) -> PeerFuture<'a, BoxedPeerConnection>;
fn listen<'a>(&'a mut self, bind_address: SocketAddr) -> PeerFuture<'a, ()>;
}
pub struct QuicPeerConnection {
connection: Connection,
remote_addr: SocketAddr,
streams: Arc<Mutex<HashMap<u64, (quinn::SendStream, quinn::RecvStream)>>>,
is_server: bool,
}
impl QuicPeerConnection {
pub fn new(connection: Connection, is_server: bool) -> Self {
let remote_addr = connection.remote_address();
Self {
connection,
remote_addr,
streams: Arc::new(Mutex::new(HashMap::new())),
is_server,
}
}
async fn get_or_create_stream(
&mut self,
stream_id: u64,
) -> Result<(quinn::SendStream, quinn::RecvStream), String> {
let mut streams = self.streams.lock().await;
if let Some((send, recv)) = streams.remove(&stream_id) {
Ok((send, recv))
} else {
drop(streams); if self.is_server {
let (send, recv) = self
.connection
.accept_bi()
.await
.map_err(|e| format!("Failed to accept bidirectional stream: {}", e))?;
Ok((send, recv))
} else {
let (send, recv) = self
.connection
.open_bi()
.await
.map_err(|e| format!("Failed to open bidirectional stream: {}", e))?;
Ok((send, recv))
}
}
}
}
impl PeerConnection for QuicPeerConnection {
fn send<'a>(&'a mut self, data: &'a [u8]) -> PeerFuture<'a, ()> {
Box::pin(async move { self.send_on_stream(0, data).await })
}
fn receive<'a>(&'a mut self) -> PeerFuture<'a, Vec<u8>> {
Box::pin(async move { self.receive_from_stream(0).await })
}
fn send_on_stream<'a>(&'a mut self, stream_id: u64, data: &'a [u8]) -> PeerFuture<'a, ()> {
Box::pin(async move {
let (mut send, recv) = self.get_or_create_stream(stream_id).await?;
send.write_all(data)
.await
.map_err(|e| format!("Failed to send data: {}", e))?;
let mut streams = self.streams.lock().await;
streams.insert(stream_id, (send, recv));
Ok(())
})
}
fn receive_from_stream<'a>(&'a mut self, stream_id: u64) -> PeerFuture<'a, Vec<u8>> {
Box::pin(async move {
let (send, mut recv) = self.get_or_create_stream(stream_id).await?;
let mut buf = vec![0u8; 65536];
match recv.read(&mut buf).await {
Ok(Some(n)) => {
buf.truncate(n);
let mut streams = self.streams.lock().await;
streams.insert(stream_id, (send, recv));
Ok(buf)
}
Ok(None) => Err("Connection closed by peer".to_string()),
Err(e) => Err(format!("Failed to receive data: {}", e)),
}
})
}
fn remote_address(&self) -> SocketAddr {
self.remote_addr
}
fn close<'a>(&'a mut self) -> PeerFuture<'a, ()> {
Box::pin(async move {
self.connection.close(0u32.into(), b"Connection closed");
Ok(())
})
}
}
#[derive(Debug, Clone)]
pub struct QuicNode {
uuid: Uuid,
party_id: PartyId,
address: SocketAddr,
}
impl QuicNode {
pub fn new_with_random_id(address: SocketAddr) -> Self {
let uuid = Uuid::new_v4();
Self {
uuid,
party_id: party_id_from_uuid(uuid),
address,
}
}
pub fn new(uuid: Uuid, address: SocketAddr) -> Self {
Self {
uuid,
party_id: party_id_from_uuid(uuid),
address,
}
}
pub fn from_party_id(id: PartyId, address: SocketAddr) -> Self {
let uuid = Uuid::from_u128(party_id_to_u128(id));
Self {
uuid,
party_id: id,
address,
}
}
pub fn address(&self) -> SocketAddr {
self.address
}
pub fn uuid(&self) -> Uuid {
self.uuid
}
}
impl Node for QuicNode {
fn id(&self) -> PartyId {
self.party_id
}
fn scalar_id<F: Field>(&self) -> F {
F::from(party_id_to_u128(self.party_id))
}
}
#[derive(Debug, Clone)]
pub struct QuicNetworkConfig {
pub timeout_ms: u64,
pub max_retries: u32,
pub use_tls: bool,
}
impl Default for QuicNetworkConfig {
fn default() -> Self {
Self {
timeout_ms: 30000, max_retries: 3,
use_tls: true,
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct QuicMessage {
sender_id: PartyId,
content: Vec<u8>,
}
impl QuicMessage {
pub fn new(sender_id: PartyId, content: Vec<u8>) -> Self {
Self { sender_id, content }
}
pub fn content(&self) -> &[u8] {
&self.content
}
}
impl Message for QuicMessage {
fn sender_id(&self) -> PartyId {
self.sender_id
}
fn bytes(&self) -> &[u8] {
&self.content
}
}
pub struct QuicNetworkManager {
endpoint: Option<Endpoint>,
#[allow(dead_code)]
server_config: Option<ServerConfig>,
#[allow(dead_code)]
client_config: Option<ClientConfig>,
nodes: Vec<QuicNode>,
node_id: PartyId,
network_config: QuicNetworkConfig,
connections: PeerConnectionMap,
}
impl Default for QuicNetworkManager {
fn default() -> Self {
Self::new()
}
}
impl QuicNetworkManager {
pub fn new() -> Self {
Self {
endpoint: None,
server_config: None,
client_config: None,
nodes: Vec::new(),
node_id: random_party_id(),
network_config: QuicNetworkConfig::default(),
connections: Arc::new(Mutex::new(HashMap::new())),
}
}
pub fn with_node_id(node_id: PartyId) -> Self {
let mut manager = Self::new();
manager.node_id = node_id;
manager
}
pub fn with_random_id() -> Self {
Self::new() }
pub fn with_config(config: QuicNetworkConfig) -> Self {
let mut manager = Self::new();
manager.network_config = config;
manager
}
fn default_transport_config() -> Result<TransportConfig, String> {
let idle_timeout = IdleTimeout::try_from(DEFAULT_IDLE_TIMEOUT)
.map_err(|e| format!("Invalid QUIC idle timeout: {e}"))?;
let mut transport = TransportConfig::default();
transport.max_concurrent_uni_streams(0u32.into());
transport.max_idle_timeout(Some(idle_timeout));
transport.keep_alive_interval(Some(DEFAULT_KEEP_ALIVE_INTERVAL));
Ok(transport)
}
pub fn local_addr(&self) -> Result<SocketAddr, String> {
self.endpoint
.as_ref()
.ok_or_else(|| {
"Endpoint not initialized. Call listen() or connect() first.".to_string()
})?
.local_addr()
.map_err(|e| format!("Failed to read endpoint local address: {e}"))
}
pub fn add_node(&mut self, node: QuicNode) {
self.nodes.push(node);
}
pub fn add_node_with_random_id(&mut self, address: SocketAddr) {
let node = QuicNode::new_with_random_id(address);
self.nodes.push(node);
}
pub fn add_node_with_uuid(&mut self, uuid: Uuid, address: SocketAddr) {
let node = QuicNode::new(uuid, address);
self.nodes.push(node);
}
pub fn add_node_with_party_id(&mut self, id: PartyId, address: SocketAddr) {
let node = QuicNode::from_party_id(id, address);
self.nodes.push(node);
}
fn create_insecure_client_config() -> Result<ClientConfig, String> {
let mut crypto = rustls::ClientConfig::builder()
.dangerous()
.with_custom_certificate_verifier(Arc::new(SkipServerVerification::new()))
.with_no_client_auth();
crypto.alpn_protocols = vec![b"quic-example".to_vec()];
let mut config = ClientConfig::new(Arc::new(
quinn::crypto::rustls::QuicClientConfig::try_from(crypto)
.map_err(|e| format!("Failed to create QUIC client config: {}", e))?,
));
config.transport_config(Arc::new(Self::default_transport_config()?));
Ok(config)
}
fn create_self_signed_server_config() -> Result<ServerConfig, String> {
let cert = rcgen::generate_simple_self_signed(vec!["localhost".into()])
.map_err(|e| format!("Failed to generate certificate: {}", e))?;
let cert_der = CertificateDer::from(cert.cert.der().to_vec());
let key_der =
PrivateKeyDer::Pkcs8(PrivatePkcs8KeyDer::from(cert.signing_key.serialize_der()));
let mut server_crypto = rustls::ServerConfig::builder()
.with_no_client_auth()
.with_single_cert(vec![cert_der], key_der)
.map_err(|e| format!("Failed to create server crypto config: {}", e))?;
server_crypto.alpn_protocols = vec![b"quic-example".to_vec()];
let mut server_config = ServerConfig::with_crypto(Arc::new(
quinn::crypto::rustls::QuicServerConfig::try_from(server_crypto)
.map_err(|e| format!("Failed to create QUIC server config: {}", e))?,
));
server_config.transport = Arc::new(Self::default_transport_config()?);
Ok(server_config)
}
}
impl NetworkManager for QuicNetworkManager {
fn connect<'a>(&'a mut self, address: SocketAddr) -> PeerFuture<'a, BoxedPeerConnection> {
Box::pin(async move {
let client_config = Self::create_insecure_client_config()?;
let connection = if let Some(endpoint) = self.endpoint.as_mut() {
endpoint.set_default_client_config(client_config.clone());
eprintln!("[quic] Using existing endpoint to connect to {}", address);
match endpoint.connect(address, "localhost") {
Ok(connecting) => {
eprintln!("[quic] Awaiting connection handshake to {}", address);
connecting.await.map_err(|e| {
format!("Failed to establish connection to {}: {}", address, e)
})?
}
Err(e) => {
eprintln!(
"[quic] Server endpoint connect failed ({}), creating client endpoint for {}",
e, address
);
let mut client_endpoint = Endpoint::client(client_endpoint_bind_addr())
.map_err(|e| format!("Failed to create client endpoint: {}", e))?;
client_endpoint.set_default_client_config(client_config);
let connecting =
client_endpoint.connect(address, "localhost").map_err(|e| {
format!("Failed to initiate connection to {}: {}", address, e)
})?;
eprintln!(
"[quic] Awaiting connection handshake to {} (client endpoint)",
address
);
connecting.await.map_err(|e| {
format!("Failed to establish connection to {}: {}", address, e)
})?
}
}
} else {
eprintln!(
"[quic] Creating new client endpoint to connect to {}",
address
);
let mut endpoint = Endpoint::client(client_endpoint_bind_addr())
.map_err(|e| format!("Failed to create client endpoint: {}", e))?;
endpoint.set_default_client_config(client_config);
let endpoint = self.endpoint.insert(endpoint);
let connecting = endpoint
.connect(address, "localhost")
.map_err(|e| format!("Failed to initiate connection to {}: {}", address, e))?;
eprintln!("[quic] Awaiting connection handshake to {}", address);
connecting
.await
.map_err(|e| format!("Failed to establish connection to {}: {}", address, e))?
};
eprintln!("[quic] Connection established to {}", address);
if let Ok((mut send, _recv)) = connection.open_bi().await {
let handshake = format!("ROLE:SERVER:{}\n", self.node_id);
let _ = send.write_all(handshake.as_bytes()).await;
}
let node_id = self
.nodes
.iter()
.find(|node| node.address() == address)
.map(|node| node.id())
.unwrap_or_else(|| {
let node = QuicNode::new_with_random_id(address);
let id = node.id();
self.nodes.push(node);
id
});
let mut connections = self.connections.lock().await;
connections.insert(
node_id,
Arc::new(Mutex::new(
Box::new(QuicPeerConnection::new(connection.clone(), false))
as Box<dyn PeerConnection>,
)),
);
Ok(Box::new(QuicPeerConnection::new(connection, false)) as Box<dyn PeerConnection>)
})
}
fn accept<'a>(&'a mut self) -> PeerFuture<'a, BoxedPeerConnection> {
Box::pin(async move {
let endpoint = self
.endpoint
.as_ref()
.ok_or_else(|| "Endpoint not initialized. Call listen() first.".to_string())?;
let incoming = endpoint
.accept()
.await
.ok_or_else(|| "No incoming connections".to_string())?;
let connection = incoming
.await
.map_err(|e| format!("Failed to accept connection: {}", e))?;
let remote_addr = connection.remote_address();
let mut parsed_id: Option<PartyId> = None;
if let Ok((mut _send, mut recv)) = connection.accept_bi().await {
let mut buf = vec![0u8; 256];
if let Ok(Some(n)) = recv.read(&mut buf).await {
let line = String::from_utf8_lossy(&buf[..n])
.lines()
.next()
.unwrap_or("")
.to_string();
if let Some(rest) = line.strip_prefix("ROLE:SERVER:") {
if let Ok(id) = rest.trim().parse::<usize>() {
parsed_id = Some(id);
}
}
}
}
let node_id = parsed_id.unwrap_or_else(|| {
let node = QuicNode::new_with_random_id(remote_addr);
let id = node.id();
self.nodes.push(node);
id
});
let mut connections = self.connections.lock().await;
connections.insert(
node_id,
Arc::new(Mutex::new(
Box::new(QuicPeerConnection::new(connection.clone(), true))
as Box<dyn PeerConnection>,
)),
);
Ok(Box::new(QuicPeerConnection::new(connection, true)) as Box<dyn PeerConnection>)
})
}
fn listen<'a>(&'a mut self, bind_address: SocketAddr) -> PeerFuture<'a, ()> {
Box::pin(async move {
let server_config = Self::create_self_signed_server_config()?;
let endpoint = Endpoint::server(server_config, bind_address)
.map_err(|e| format!("Failed to create server endpoint: {}", e))?;
self.endpoint = Some(endpoint);
Ok(())
})
}
}
#[async_trait]
impl Network for QuicNetworkManager {
type NodeType = QuicNode;
type NetworkConfig = QuicNetworkConfig;
async fn send(&self, recipient: PartyId, message: &[u8]) -> Result<usize, NetworkError> {
let connection_arc = {
let connections = self.connections.lock().await;
connections.get(&recipient).cloned()
};
let connection_arc = connection_arc.ok_or(NetworkError::PartyNotFound(recipient))?;
let preview_len = message.len().min(64);
let hex_preview: String = message[..preview_len]
.iter()
.map(|b| format!("{:02x}", b))
.collect();
debug!(
"[MSG:SEND] from node {} to node {} ({} bytes, hex[0..{}]={})",
self.node_id,
recipient,
message.len(),
preview_len,
hex_preview
);
let mut connection = connection_arc.lock().await;
match connection.send(message).await {
Ok(_) => {
debug!(
"[MSG:SENT] from node {} to node {} ({} bytes)",
self.node_id,
recipient,
message.len()
);
Ok(message.len())
}
Err(e) => {
debug!(
"[MSG:SEND-FAIL] from node {} to node {}: {}",
self.node_id, recipient, e
);
Err(NetworkError::SendError)
}
}
}
async fn broadcast(&self, message: &[u8]) -> Result<usize, NetworkError> {
let mut total_bytes = 0;
let preview_len = message.len().min(64);
let hex_preview: String = message[..preview_len]
.iter()
.map(|b| format!("{:02x}", b))
.collect();
debug!(
"[MSG:BROADCAST] from node {} to all ({} bytes, hex[0..{}]={})",
self.node_id,
message.len(),
preview_len,
hex_preview
);
let targets: Vec<(PartyId, Arc<Mutex<Box<dyn PeerConnection>>>)> = {
let connections = self.connections.lock().await;
self.nodes
.iter()
.filter(|node| node.id() != self.node_id)
.filter_map(|node| {
connections
.get(&node.id())
.map(|conn| (node.id(), conn.clone()))
})
.collect()
};
for node in &self.nodes {
if node.id() != self.node_id && !targets.iter().any(|(id, _)| *id == node.id()) {
debug!(
"[MSG:BROADCAST-SKIP] from node {} -> node {}: no connection",
self.node_id,
node.id()
);
}
}
for (node_id, connection_arc) in targets {
let mut connection = connection_arc.lock().await;
match connection.send(message).await {
Ok(_) => {
debug!(
"[MSG:BROADCAST-SENT] from node {} -> node {} ({} bytes)",
self.node_id,
node_id,
message.len()
);
total_bytes += message.len();
}
Err(e) => {
debug!(
"[MSG:BROADCAST-FAIL] from node {} -> node {}: {}",
self.node_id, node_id, e
);
}
}
}
if total_bytes > 0 {
Ok(total_bytes)
} else {
Err(NetworkError::SendError)
}
}
fn parties(&self) -> Vec<&Self::NodeType> {
self.nodes.iter().collect()
}
fn parties_mut(&mut self) -> Vec<&mut Self::NodeType> {
self.nodes.iter_mut().collect()
}
fn config(&self) -> &Self::NetworkConfig {
&self.network_config
}
fn node(&self, id: PartyId) -> Option<&Self::NodeType> {
self.nodes.iter().find(|node| node.id() == id)
}
fn node_mut(&mut self, id: PartyId) -> Option<&mut Self::NodeType> {
self.nodes.iter_mut().find(|node| node.id() == id)
}
async fn send_to_client(
&self,
client: ClientId,
_message: &[u8],
) -> Result<usize, NetworkError> {
Err(NetworkError::ClientNotFound(client))
}
fn clients(&self) -> Vec<ClientId> {
Vec::new()
}
fn is_client_connected(&self, _client: ClientId) -> bool {
false
}
fn local_party_id(&self) -> PartyId {
self.node_id
}
fn party_count(&self) -> usize {
self.nodes.len()
}
fn verified_ordering(&self) -> Option<VerifiedOrdering> {
None
}
}
#[cfg(test)]
mod tests {
use super::*;
fn loopback_addr() -> SocketAddr {
SocketAddr::from(([127, 0, 0, 1], 0))
}
#[test]
fn client_endpoint_bind_address_is_unspecified_ephemeral_ipv4() {
let addr = client_endpoint_bind_addr();
assert!(addr.ip().is_unspecified());
assert_eq!(addr.port(), 0);
}
#[test]
fn default_transport_config_is_constructible_without_panics() {
QuicNetworkManager::default_transport_config()
.expect("valid default QUIC transport config");
}
#[test]
fn quic_node_from_party_id_preserves_explicit_party_id() {
let node = QuicNode::from_party_id(PartyId::MAX, loopback_addr());
assert_eq!(node.id(), PartyId::MAX);
}
#[test]
fn quic_node_uuid_constructor_uses_explicit_uuid_fold() {
let uuid = Uuid::from_u128(party_id_to_u128(PartyId::MAX) + 1);
let node = QuicNode::new(uuid, loopback_addr());
assert_eq!(node.uuid(), uuid);
assert_eq!(node.id(), party_id_from_uuid(uuid));
}
#[test]
fn quic_node_scalar_id_uses_party_id() {
let node = QuicNode::from_party_id(42, loopback_addr());
assert_eq!(
node.scalar_id::<ark_bn254::Fr>(),
ark_bn254::Fr::from(42u128)
);
}
#[test]
fn local_addr_reports_uninitialized_endpoint() {
let manager = QuicNetworkManager::new();
let err = manager
.local_addr()
.expect_err("endpoint is not initialized");
assert!(err.contains("Endpoint not initialized"));
}
}
#[derive(Debug)]
struct SkipServerVerification;
impl SkipServerVerification {
fn new() -> Self {
Self
}
}
impl rustls::client::danger::ServerCertVerifier for SkipServerVerification {
fn verify_server_cert(
&self,
_end_entity: &CertificateDer<'_>,
_intermediates: &[CertificateDer<'_>],
_server_name: &rustls::pki_types::ServerName<'_>,
_ocsp: &[u8],
_now: rustls::pki_types::UnixTime,
) -> Result<rustls::client::danger::ServerCertVerified, rustls::Error> {
Ok(rustls::client::danger::ServerCertVerified::assertion())
}
fn verify_tls12_signature(
&self,
message: &[u8],
cert: &CertificateDer<'_>,
dss: &rustls::DigitallySignedStruct,
) -> Result<rustls::client::danger::HandshakeSignatureValid, rustls::Error> {
rustls::crypto::verify_tls12_signature(
message,
cert,
dss,
&rustls::crypto::ring::default_provider().signature_verification_algorithms,
)
}
fn verify_tls13_signature(
&self,
message: &[u8],
cert: &CertificateDer<'_>,
dss: &rustls::DigitallySignedStruct,
) -> Result<rustls::client::danger::HandshakeSignatureValid, rustls::Error> {
rustls::crypto::verify_tls13_signature(
message,
cert,
dss,
&rustls::crypto::ring::default_provider().signature_verification_algorithms,
)
}
fn supported_verify_schemes(&self) -> Vec<rustls::SignatureScheme> {
rustls::crypto::ring::default_provider()
.signature_verification_algorithms
.supported_schemes()
}
}