use super::*;
#[inline(always)]
pub(crate) fn route_quic_datagram(packet: &[u8], shard_count: usize) -> usize {
if shard_count <= 1 || packet.is_empty() {
return 0;
}
let first_cid_byte: u8 = if (packet[0] & 0x80) != 0 {
let dcid_len: usize = packet.get(5).copied().unwrap_or(0) as usize;
if dcid_len > 0 {
packet.get(6).copied().unwrap_or(0)
} else {
0
}
} else {
packet.get(1).copied().unwrap_or(0)
};
usize::from(first_cid_byte) % shard_count
}
async fn recv_udp_batch(
real_socket: &Arc<dyn AsyncUdpSocket>,
buffers: &mut [Vec<u8>],
metas: &mut [RecvMeta],
) -> io::Result<usize> {
poll_fn(|cx: &mut task::Context<'_>| {
let mut slices: [MaybeUninit<IoSliceMut<'_>>; BATCH_SIZE] =
[const { MaybeUninit::uninit() }; BATCH_SIZE];
let count: usize = buffers.len().min(BATCH_SIZE).min(metas.len());
for (idx, buf) in buffers[..count].iter_mut().enumerate() {
slices[idx].write(IoSliceMut::new(buf.as_mut_slice()));
}
let slice_refs: &mut [IoSliceMut<'_>] = unsafe {
slice::from_raw_parts_mut(slices.as_mut_ptr().cast::<IoSliceMut<'_>>(), count)
};
real_socket.poll_recv(cx, slice_refs, &mut metas[..count])
})
.await
}
async fn run_quic_shard_dispatcher(
real_socket: Arc<dyn AsyncUdpSocket>,
queues: Vec<Arc<Mutex<ShardQueueState>>>,
mut shutdown_rx: Receiver<()>,
) {
let seg_count: usize = real_socket.max_receive_segments().max(1);
let buf_size: usize = (seg_count * 1500).max(64 * 1024);
let mut buffers: Vec<Vec<u8>> = (0..BATCH_SIZE).map(|_: usize| vec![0; buf_size]).collect();
let mut metas: Vec<RecvMeta> = vec![RecvMeta::default(); BATCH_SIZE];
let shard_count: usize = queues.len();
let mut wakers_to_wake: [Option<Waker>; 64] = [const { None }; 64];
let mut staged: Vec<Vec<(usize, usize, usize, RecvMeta)>> = (0..shard_count)
.map(|_: usize| Vec::with_capacity(BATCH_SIZE))
.collect();
loop {
let batch_res: Option<io::Result<usize>> = tokio::select! {
biased;
_ = shutdown_rx.changed() => None,
received = recv_udp_batch(&real_socket, &mut buffers, &mut metas) => Some(received),
};
let Some(received) = batch_res else {
break;
};
let Ok(count) = received else {
continue;
};
let mut touched_mask: u64 = 0;
for idx in 0..count {
let meta: RecvMeta = metas[idx];
let total_len: usize = meta.len.min(buffers[idx].len());
if total_len == 0 {
continue;
}
let stride: usize = meta.stride.max(1);
let mut offset: usize = 0;
while offset < total_len {
let end: usize = (offset + stride).min(total_len);
let slice: &[u8] = &buffers[idx][offset..end];
let shard_idx: usize = route_quic_datagram(slice, shard_count);
let mut seg_meta: RecvMeta = meta;
seg_meta.len = slice.len();
seg_meta.stride = slice.len();
staged[shard_idx].push((idx, offset, end, seg_meta));
if shard_idx < 64 {
touched_mask |= 1u64 << shard_idx;
}
offset = end;
}
}
let mut wake_mask: u64 = 0;
let mut iter_mask: u64 = touched_mask;
while iter_mask != 0 {
let shard_idx: usize = iter_mask.trailing_zeros() as usize;
iter_mask &= iter_mask - 1;
let batch: &mut Vec<(usize, usize, usize, RecvMeta)> = &mut staged[shard_idx];
if batch.is_empty() {
continue;
}
if let Ok(mut q) = queues[shard_idx].lock() {
for &(buf_idx, start, end, seg_meta) in batch.iter() {
if q.queue.len() >= QUIC_SHARD_QUEUE_CAP {
break;
}
let slice: &[u8] = &buffers[buf_idx][start..end];
let mut payload: Vec<u8> = q.pool.pop().unwrap_or_else(|| {
Vec::with_capacity(slice.len().max(QUIC_DATAGRAM_INIT_CAP))
});
payload.clear();
payload.extend_from_slice(slice);
q.queue.push_back(ShardDatagram {
data: payload,
meta: seg_meta,
});
}
if let Some(waker) = q.waker.take() {
wakers_to_wake[shard_idx] = Some(waker);
wake_mask |= 1u64 << shard_idx;
}
}
batch.clear();
}
let mut mask: u64 = wake_mask;
while mask != 0 {
let bit: usize = mask.trailing_zeros() as usize;
mask &= mask - 1;
if let Some(waker) = wakers_to_wake[bit].take() {
waker.wake();
}
}
}
for queue in queues {
let waker: Option<Waker> =
queue
.lock()
.ok()
.and_then(|mut q: MutexGuard<'_, ShardQueueState>| {
q.closed = true;
q.waker.take()
});
if let Some(waker) = waker {
waker.wake();
}
}
}
pub(crate) fn create_quic_endpoint_set(
quic_config: Arc<QuicServerConfig>,
bind_addr: SocketAddr,
) -> Result<QuicEndpointSet, Box<ServerError>> {
let workers: usize = Handle::current().metrics().num_workers();
if workers <= 1 {
let domain: Domain = if bind_addr.is_ipv6() {
Domain::IPV6
} else {
Domain::IPV4
};
let socket: Socket = Socket::new(domain, Type::DGRAM, None)
.map_err(|error: io::Error| Box::new(ServerError::from(error)))?;
let _: io::Result<()> = socket.set_recv_buffer_size(QUIC_SHARD_SOCKET_BUF_SIZE);
let _: io::Result<()> = socket.set_send_buffer_size(QUIC_SHARD_SOCKET_BUF_SIZE);
socket
.bind(&bind_addr.into())
.map_err(|error: io::Error| Box::new(ServerError::from(error)))?;
socket
.set_nonblocking(true)
.map_err(|error: io::Error| Box::new(ServerError::from(error)))?;
let std_udp: StdUdpSocket = StdUdpSocket::from(socket);
let endpoint: Endpoint = Endpoint::new(
EndpointConfig::default(),
Some(quinn::ServerConfig::with_crypto(quic_config)),
std_udp,
Arc::new(TokioRuntime),
)
.map_err(|error: io::Error| Box::new(ServerError::from(error)))?;
return Ok(QuicEndpointSet {
endpoints: vec![endpoint],
dispatcher: None,
dispatcher_stop: None,
});
}
let shard_count: usize = workers.min(64);
let std_udp: StdUdpSocket = StdUdpSocket::bind(bind_addr)
.map_err(|error: io::Error| Box::new(ServerError::from(error)))?;
let sock_ref: SockRef<'_> = SockRef::from(&std_udp);
let _: Result<(), io::Error> = sock_ref.set_recv_buffer_size(QUIC_SHARD_SOCKET_BUF_SIZE);
let _: Result<(), io::Error> = sock_ref.set_send_buffer_size(QUIC_SHARD_SOCKET_BUF_SIZE);
let rx_udp: StdUdpSocket = std_udp
.try_clone()
.map_err(|error: io::Error| Box::new(ServerError::from(error)))?;
let runtime: Arc<dyn Runtime> = Arc::new(TokioRuntime);
let tx_socket: Arc<dyn AsyncUdpSocket> = runtime
.wrap_udp_socket(std_udp)
.map_err(|error: io::Error| Box::new(ServerError::from(error)))?;
let mut queues: Vec<Arc<Mutex<ShardQueueState>>> = Vec::with_capacity(shard_count);
let mut endpoints: Vec<Endpoint> = Vec::with_capacity(shard_count);
for shard_idx in 0..shard_count {
let queue: Arc<Mutex<ShardQueueState>> = Arc::new(Mutex::new(ShardQueueState {
queue: VecDeque::new(),
pool: Vec::new(),
waker: None,
closed: false,
}));
let shard_socket: Arc<ShardUdpSocket> = Arc::new(ShardUdpSocket {
real_socket: Arc::clone(&tx_socket),
queue: Arc::clone(&queue),
});
let mut endpoint_config: EndpointConfig = EndpointConfig::default();
let shard_id: u8 = shard_idx as u8;
let total_shards: u8 = shard_count as u8;
endpoint_config.cid_generator(move || {
Box::new(ShardCidGenerator {
shard_id,
shard_count: total_shards,
})
});
let crypto: Arc<QuicServerConfig> = Arc::clone(&quic_config);
let endpoint: Endpoint = Endpoint::new_with_abstract_socket(
endpoint_config,
Some(quinn::ServerConfig::with_crypto(crypto)),
shard_socket,
Arc::clone(&runtime),
)
.map_err(|error: io::Error| Box::new(ServerError::from(error)))?;
queues.push(queue);
endpoints.push(endpoint);
}
let (stop_tx, stop_rx): (Sender<()>, Receiver<()>) = channel(());
let dispatcher: ThreadJoinHandle<()> = ThreadBuilder::new()
.name(QUIC_DISPATCHER_THREAD_NAME.to_owned())
.spawn(move || {
let Ok(rx_runtime) = RuntimeBuilder::new_current_thread().enable_io().build() else {
return;
};
rx_runtime.block_on(async move {
let rx_quinn_rt: TokioRuntime = TokioRuntime;
let Ok(rx_socket) = rx_quinn_rt.wrap_udp_socket(rx_udp) else {
return;
};
run_quic_shard_dispatcher(rx_socket, queues, stop_rx).await;
});
})
.map_err(|error: io::Error| Box::new(ServerError::from(error)))?;
Ok(QuicEndpointSet {
endpoints,
dispatcher: Some(dispatcher),
dispatcher_stop: Some(stop_tx),
})
}