use std::collections::{BTreeMap, HashMap, VecDeque};
use std::net::SocketAddr;
use std::pin::Pin;
use std::sync::Arc;
use std::task::{Context, Poll};
use std::time::{Duration, Instant};
use tokio::io::{AsyncRead, AsyncWrite, ReadBuf};
use tokio::net::UdpSocket;
use tokio::sync::{mpsc, Mutex};
use tokio::time::timeout;
use crate::error::{Error, Result};
use crate::utp::packet::{
build_selective_ack, iter_selective_ack, now_micros, Packet, PacketType,
};
const MAX_PAYLOAD: usize = 1400;
const DEFAULT_WND: u32 = 1_048_576;
const INIT_CWND: u32 = MAX_PAYLOAD as u32 * 2;
const MIN_CWND: u32 = MAX_PAYLOAD as u32;
const MAX_CWND: u32 = MAX_PAYLOAD as u32 * 64;
const HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(10);
const RTO_FLOOR: Duration = Duration::from_millis(200);
const RTO_CEIL: Duration = Duration::from_secs(3);
struct DemuxState {
routes: HashMap<u16, mpsc::UnboundedSender<(SocketAddr, Packet)>>,
syn_tx: Option<mpsc::UnboundedSender<(SocketAddr, Packet)>>,
}
pub struct UtpEndpoint {
socket: Arc<UdpSocket>,
local_addr: SocketAddr,
demux: Arc<Mutex<DemuxState>>,
}
impl UtpEndpoint {
pub fn local_addr(&self) -> SocketAddr {
self.local_addr
}
pub fn socket(&self) -> Arc<UdpSocket> {
Arc::clone(&self.socket)
}
}
pub async fn bind_utp(addr: SocketAddr) -> Result<UtpEndpoint> {
let socket = UdpSocket::bind(addr).await.map_err(Error::from)?;
let local_addr = socket.local_addr().map_err(Error::from)?;
let socket = Arc::new(socket);
let demux = Arc::new(Mutex::new(DemuxState {
routes: HashMap::new(),
syn_tx: None,
}));
spawn_demux(Arc::clone(&socket), Arc::clone(&demux));
Ok(UtpEndpoint {
socket,
local_addr,
demux,
})
}
fn spawn_demux(socket: Arc<UdpSocket>, demux: Arc<Mutex<DemuxState>>) {
tokio::spawn(async move {
let mut buf = vec![0u8; 2048];
loop {
let Ok((n, peer)) = socket.recv_from(&mut buf).await else {
break;
};
let packet = match Packet::decode(&buf[..n]) {
Ok(p) => p,
Err(_) => continue,
};
let guard = demux.lock().await;
if packet.ty == PacketType::Syn {
if let Some(tx) = &guard.syn_tx {
let _ = tx.send((peer, packet));
}
continue;
}
if let Some(tx) = guard.routes.get(&packet.connection_id) {
let _ = tx.send((peer, packet));
}
}
});
}
struct StreamBootstrap {
socket: Arc<UdpSocket>,
peer: SocketAddr,
demux: Arc<Mutex<DemuxState>>,
conn_id_send: u16,
conn_id_recv: u16,
seq_nr: u16,
ack_nr: u16,
inbound: mpsc::UnboundedReceiver<(SocketAddr, Packet)>,
last_peer_ts: u32,
}
fn new_stream(boot: StreamBootstrap) -> UtpStream {
UtpStream {
socket: boot.socket,
peer: boot.peer,
demux: boot.demux,
conn_id_send: boot.conn_id_send,
conn_id_recv: boot.conn_id_recv,
seq_nr: boot.seq_nr,
ack_nr: boot.ack_nr,
inbound: boot.inbound,
read_buf: VecDeque::new(),
reorder: BTreeMap::new(),
in_flight: VecDeque::new(),
closed: false,
last_peer_ts: boot.last_peer_ts,
peer_wnd: DEFAULT_WND,
cwnd: INIT_CWND,
srtt_us: 100_000,
rttvar_us: 50_000,
}
}
pub async fn connect(endpoint: &UtpEndpoint, peer: SocketAddr) -> Result<UtpStream> {
let mut conn_seed = [0u8; 2];
getrandom::fill(&mut conn_seed).map_err(|err| Error::Secure(format!("uTP CSPRNG: {err}")))?;
let conn_id_recv = u16::from_be_bytes(conn_seed);
let conn_id_send = conn_id_recv.wrapping_add(1);
let (tx, mut rx) = mpsc::unbounded_channel();
{
let mut guard = endpoint.demux.lock().await;
guard.routes.insert(conn_id_recv, tx);
}
let mut seq_nr: u16 = 1;
let syn = Packet {
ty: PacketType::Syn,
connection_id: conn_id_recv,
timestamp_microseconds: now_micros(),
timestamp_difference_microseconds: 0,
wnd_size: DEFAULT_WND,
seq_nr,
ack_nr: 0,
selective_ack: None,
payload: Vec::new(),
};
endpoint
.socket
.send_to(&syn.encode(), peer)
.await
.map_err(Error::from)?;
let (_from, reply) = timeout(HANDSHAKE_TIMEOUT, recv_matching(&mut rx, |p| {
p.ty == PacketType::State && p.connection_id == conn_id_recv
}))
.await
.map_err(|_| Error::Peer("uTP connect timed out".into()))??;
seq_nr = seq_nr.wrapping_add(1);
Ok(new_stream(StreamBootstrap {
socket: endpoint.socket(),
peer,
demux: Arc::clone(&endpoint.demux),
conn_id_send,
conn_id_recv,
seq_nr,
ack_nr: reply.seq_nr,
inbound: rx,
last_peer_ts: reply.timestamp_microseconds,
}))
}
pub async fn accept(endpoint: &UtpEndpoint) -> Result<UtpStream> {
let (syn_tx, mut syn_rx) = mpsc::unbounded_channel();
{
let mut guard = endpoint.demux.lock().await;
guard.syn_tx = Some(syn_tx);
}
let (peer, syn) = timeout(HANDSHAKE_TIMEOUT, syn_rx.recv())
.await
.map_err(|_| Error::Peer("uTP accept timed out".into()))?
.ok_or_else(|| Error::Peer("uTP accept channel closed".into()))?;
{
let mut guard = endpoint.demux.lock().await;
guard.syn_tx = None;
}
let conn_id_send = syn.connection_id;
let conn_id_recv = syn.connection_id.wrapping_add(1);
let (tx, rx) = mpsc::unbounded_channel();
{
let mut guard = endpoint.demux.lock().await;
guard.routes.insert(conn_id_recv, tx);
}
let seq_nr: u16 = 1;
let ack_nr = syn.seq_nr;
let state = Packet {
ty: PacketType::State,
connection_id: conn_id_send,
timestamp_microseconds: now_micros(),
timestamp_difference_microseconds: now_micros().wrapping_sub(syn.timestamp_microseconds),
wnd_size: DEFAULT_WND,
seq_nr,
ack_nr,
selective_ack: None,
payload: Vec::new(),
};
endpoint
.socket
.send_to(&state.encode(), peer)
.await
.map_err(Error::from)?;
Ok(new_stream(StreamBootstrap {
socket: endpoint.socket(),
peer,
demux: Arc::clone(&endpoint.demux),
conn_id_send,
conn_id_recv,
seq_nr: seq_nr.wrapping_add(1),
ack_nr,
inbound: rx,
last_peer_ts: syn.timestamp_microseconds,
}))
}
async fn recv_matching(
rx: &mut mpsc::UnboundedReceiver<(SocketAddr, Packet)>,
pred: impl Fn(&Packet) -> bool,
) -> Result<(SocketAddr, Packet)> {
while let Some((addr, packet)) = rx.recv().await {
if pred(&packet) {
return Ok((addr, packet));
}
}
Err(Error::Peer("uTP connection closed during handshake".into()))
}
struct InFlight {
seq: u16,
payload: Vec<u8>,
sent_at: Instant,
retries: u8,
}
pub struct UtpStream {
socket: Arc<UdpSocket>,
peer: SocketAddr,
demux: Arc<Mutex<DemuxState>>,
conn_id_send: u16,
conn_id_recv: u16,
seq_nr: u16,
ack_nr: u16,
inbound: mpsc::UnboundedReceiver<(SocketAddr, Packet)>,
read_buf: VecDeque<u8>,
reorder: BTreeMap<u16, Vec<u8>>,
in_flight: VecDeque<InFlight>,
closed: bool,
last_peer_ts: u32,
peer_wnd: u32,
cwnd: u32,
srtt_us: u32,
rttvar_us: u32,
}
impl UtpStream {
pub fn peer_addr(&self) -> SocketAddr {
self.peer
}
fn rto(&self) -> Duration {
let us = self
.srtt_us
.saturating_add(self.rttvar_us.saturating_mul(4))
.max(RTO_FLOOR.as_micros() as u32)
.min(RTO_CEIL.as_micros() as u32);
Duration::from_micros(u64::from(us))
}
fn in_flight_bytes(&self) -> u32 {
self.in_flight
.iter()
.map(|p| p.payload.len() as u32)
.fold(0u32, u32::saturating_add)
}
fn send_budget(&self) -> u32 {
let limit = self.cwnd.min(self.peer_wnd).max(MIN_CWND);
limit.saturating_sub(self.in_flight_bytes())
}
async fn send_packet(&self, mut packet: Packet) -> Result<()> {
packet.timestamp_microseconds = now_micros();
packet.timestamp_difference_microseconds =
now_micros().wrapping_sub(self.last_peer_ts);
packet.wnd_size = DEFAULT_WND;
self.socket
.send_to(&packet.encode(), self.peer)
.await
.map_err(Error::from)?;
Ok(())
}
fn update_rtt(&mut self, sample_us: u32) {
let err = sample_us as i64 - self.srtt_us as i64;
self.srtt_us = (self.srtt_us as i64 + err / 8).max(1) as u32;
self.rttvar_us = (self.rttvar_us as i64 + (err.abs() - self.rttvar_us as i64) / 4)
.max(1) as u32;
}
fn on_ack_progress(&mut self) {
self.cwnd = (self.cwnd.saturating_add(MAX_PAYLOAD as u32 / 4)).min(MAX_CWND);
}
fn on_loss(&mut self) {
self.cwnd = (self.cwnd / 2).max(MIN_CWND);
}
fn apply_cumulative_ack(&mut self, ack_nr: u16) {
let mut progressed = false;
while let Some(front) = self.in_flight.front() {
let dist = ack_nr.wrapping_sub(front.seq);
if dist < 0x8000 {
let sample = front.sent_at.elapsed().as_micros().min(u128::from(u32::MAX)) as u32;
self.update_rtt(sample);
self.in_flight.pop_front();
progressed = true;
} else {
break;
}
}
if progressed {
self.on_ack_progress();
}
}
fn apply_selective_ack(&mut self, ack_nr: u16, bitmask: &[u8]) {
let sacked: std::collections::HashSet<u16> =
iter_selective_ack(ack_nr, bitmask).collect();
if sacked.is_empty() {
return;
}
let before = self.in_flight.len();
self.in_flight.retain(|p| !sacked.contains(&p.seq));
if self.in_flight.len() < before {
self.on_ack_progress();
}
}
fn sack_bitmask(&self) -> Option<Vec<u8>> {
build_selective_ack(self.ack_nr, self.reorder.keys().copied())
}
async fn send_state_ack(&mut self) -> Result<()> {
let state = Packet {
ty: PacketType::State,
connection_id: self.conn_id_send,
timestamp_microseconds: 0,
timestamp_difference_microseconds: 0,
wnd_size: DEFAULT_WND,
seq_nr: self.seq_nr,
ack_nr: self.ack_nr,
selective_ack: self.sack_bitmask(),
payload: Vec::new(),
};
self.send_packet(state).await
}
fn deliver_in_order(&mut self, seq: u16, payload: Vec<u8>) {
let expected = self.ack_nr.wrapping_add(1);
if seq == expected {
self.read_buf.extend(payload);
self.ack_nr = seq;
while let Some(next) = self.reorder.remove(&self.ack_nr.wrapping_add(1)) {
self.ack_nr = self.ack_nr.wrapping_add(1);
self.read_buf.extend(next);
}
} else if seq.wrapping_sub(self.ack_nr) > 1 && seq.wrapping_sub(self.ack_nr) < 0x8000 {
self.reorder.insert(seq, payload);
}
}
async fn handle_incoming(&mut self, packet: Packet) -> Result<()> {
self.last_peer_ts = packet.timestamp_microseconds;
self.peer_wnd = packet.wnd_size.max(MIN_CWND);
match packet.ty {
PacketType::Data => {
self.deliver_in_order(packet.seq_nr, packet.payload);
self.send_state_ack().await?;
}
PacketType::Fin => {
self.deliver_in_order(packet.seq_nr, Vec::new());
self.closed = true;
let _ = self.send_state_ack().await;
}
PacketType::Reset => {
self.closed = true;
}
PacketType::State => {
self.apply_cumulative_ack(packet.ack_nr);
if let Some(sack) = packet.selective_ack.as_deref() {
self.apply_selective_ack(packet.ack_nr, sack);
}
}
PacketType::Syn => {}
}
Ok(())
}
async fn pump(&mut self) -> Result<()> {
while let Ok((addr, packet)) = self.inbound.try_recv() {
if addr != self.peer {
continue;
}
self.handle_incoming(packet).await?;
}
Ok(())
}
async fn retransmit_oldest(&mut self) -> Result<()> {
let Some(front) = self.in_flight.front() else {
return Ok(());
};
let seq = front.seq;
let payload = front.payload.clone();
let sack = self.sack_bitmask();
let ack_nr = self.ack_nr;
let conn_id = self.conn_id_send;
if let Some(front) = self.in_flight.front_mut() {
front.retries = front.retries.saturating_add(1);
front.sent_at = Instant::now();
}
let packet = Packet {
ty: PacketType::Data,
connection_id: conn_id,
timestamp_microseconds: 0,
timestamp_difference_microseconds: 0,
wnd_size: DEFAULT_WND,
seq_nr: seq,
ack_nr,
selective_ack: sack,
payload,
};
self.on_loss();
self.send_packet(packet).await
}
pub async fn read(&mut self, buf: &mut [u8]) -> Result<usize> {
loop {
self.pump().await?;
if !self.read_buf.is_empty() {
let n = buf.len().min(self.read_buf.len());
for slot in buf.iter_mut().take(n) {
*slot = self.read_buf.pop_front().expect("buf");
}
return Ok(n);
}
if self.closed {
return Ok(0);
}
let (addr, packet) = timeout(Duration::from_secs(120), self.inbound.recv())
.await
.map_err(|_| Error::Peer("uTP read timed out".into()))?
.ok_or_else(|| Error::Peer("uTP connection closed".into()))?;
if addr != self.peer {
continue;
}
self.handle_incoming(packet).await?;
}
}
pub async fn read_exact(&mut self, buf: &mut [u8]) -> Result<()> {
let mut filled = 0;
while filled < buf.len() {
let n = self.read(&mut buf[filled..]).await?;
if n == 0 {
return Err(Error::Peer("uTP connection closed".into()));
}
filled += n;
}
Ok(())
}
pub async fn write_all(&mut self, mut data: &[u8]) -> Result<()> {
while !data.is_empty() || !self.in_flight.is_empty() {
while !data.is_empty() {
let budget = self.send_budget();
if budget < 1 {
break;
}
let chunk_len = data.len().min(MAX_PAYLOAD).min(budget as usize);
if chunk_len == 0 {
break;
}
let chunk = data[..chunk_len].to_vec();
let seq = self.seq_nr;
let packet = Packet {
ty: PacketType::Data,
connection_id: self.conn_id_send,
timestamp_microseconds: 0,
timestamp_difference_microseconds: 0,
wnd_size: DEFAULT_WND,
seq_nr: seq,
ack_nr: self.ack_nr,
selective_ack: self.sack_bitmask(),
payload: chunk.clone(),
};
self.send_packet(packet).await?;
self.in_flight.push_back(InFlight {
seq,
payload: chunk,
sent_at: Instant::now(),
retries: 0,
});
self.seq_nr = self.seq_nr.wrapping_add(1);
data = &data[chunk_len..];
}
if self.in_flight.is_empty() {
break;
}
let wait = self.rto();
let oldest = self.in_flight.front().map(|p| p.sent_at).unwrap_or_else(Instant::now);
let remaining = wait.saturating_sub(oldest.elapsed());
match timeout(remaining.max(Duration::from_millis(1)), self.inbound.recv()).await {
Ok(Some((addr, pkt))) => {
if addr == self.peer {
self.handle_incoming(pkt).await?;
}
}
Ok(None) => return Err(Error::Peer("uTP closed while writing".into())),
Err(_) => {
if let Some(front) = self.in_flight.front() {
if front.retries >= 8 {
return Err(Error::Peer("uTP retransmit limit exceeded".into()));
}
}
self.retransmit_oldest().await?;
}
}
self.pump().await?;
}
Ok(())
}
}
impl Drop for UtpStream {
fn drop(&mut self) {
let demux = Arc::clone(&self.demux);
let id = self.conn_id_recv;
let socket = Arc::clone(&self.socket);
let peer = self.peer;
let conn_id_send = self.conn_id_send;
let seq_nr = self.seq_nr;
let ack_nr = self.ack_nr;
tokio::spawn(async move {
let mut guard = demux.lock().await;
guard.routes.remove(&id);
drop(guard);
let fin = Packet {
ty: PacketType::Fin,
connection_id: conn_id_send,
timestamp_microseconds: now_micros(),
timestamp_difference_microseconds: 0,
wnd_size: DEFAULT_WND,
seq_nr,
ack_nr,
selective_ack: None,
payload: Vec::new(),
};
let _ = socket.send_to(&fin.encode(), peer).await;
});
}
}
impl AsyncRead for UtpStream {
fn poll_read(
mut self: Pin<&mut Self>,
cx: &mut Context<'_>,
buf: &mut ReadBuf<'_>,
) -> Poll<std::io::Result<()>> {
if !self.read_buf.is_empty() {
let n = buf.remaining().min(self.read_buf.len());
let mut tmp = vec![0u8; n];
for slot in tmp.iter_mut() {
*slot = self.read_buf.pop_front().expect("buf");
}
buf.put_slice(&tmp);
return Poll::Ready(Ok(()));
}
match self.inbound.poll_recv(cx) {
Poll::Ready(Some((addr, packet))) => {
if addr == self.peer {
self.last_peer_ts = packet.timestamp_microseconds;
self.peer_wnd = packet.wnd_size.max(MIN_CWND);
match packet.ty {
PacketType::Data => {
self.deliver_in_order(packet.seq_nr, packet.payload);
let socket = Arc::clone(&self.socket);
let peer = self.peer;
let state = Packet {
ty: PacketType::State,
connection_id: self.conn_id_send,
timestamp_microseconds: now_micros(),
timestamp_difference_microseconds: 0,
wnd_size: DEFAULT_WND,
seq_nr: self.seq_nr,
ack_nr: self.ack_nr,
selective_ack: self.sack_bitmask(),
payload: Vec::new(),
};
tokio::spawn(async move {
let _ = socket.send_to(&state.encode(), peer).await;
});
}
PacketType::State => {
self.apply_cumulative_ack(packet.ack_nr);
if let Some(sack) = packet.selective_ack.as_deref() {
self.apply_selective_ack(packet.ack_nr, sack);
}
}
PacketType::Fin | PacketType::Reset => self.closed = true,
PacketType::Syn => {}
}
}
if !self.read_buf.is_empty() {
let n = buf.remaining().min(self.read_buf.len());
let mut tmp = vec![0u8; n];
for slot in tmp.iter_mut() {
*slot = self.read_buf.pop_front().expect("buf");
}
buf.put_slice(&tmp);
return Poll::Ready(Ok(()));
}
if self.closed {
return Poll::Ready(Ok(()));
}
cx.waker().wake_by_ref();
Poll::Pending
}
Poll::Ready(None) => Poll::Ready(Ok(())),
Poll::Pending => Poll::Pending,
}
}
}
impl AsyncWrite for UtpStream {
fn poll_write(
self: Pin<&mut Self>,
_cx: &mut Context<'_>,
_buf: &[u8],
) -> Poll<std::io::Result<usize>> {
Poll::Ready(Err(std::io::Error::new(
std::io::ErrorKind::Unsupported,
"UtpStream: use write_all",
)))
}
fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
Poll::Ready(Ok(()))
}
fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
Poll::Ready(Ok(()))
}
}