//! Real native-QUIC connect/accept + 1-RTT UDP data-plane pump for ATP-over-QUIC.
//!
//! This module is the wiring that turns [`super::send_path`] / the native receive
//! entry from a fail-closed scaffold into an actual transfer over a real UDP
//! socket. It composes three already-landed, separately-proven pieces — none of
//! which previously talked to each other in production — entirely over their
//! public APIs, so no peer-owned Phase-A file
//! (`connection.rs`/`connection_manager.rs`/`managed_endpoint.rs`) is modified:
//!
//! 1. [`QuicHandshakeDriver`] runs the genuine `rustls::quic` TLS-1.3 handshake
//! over a [`QuicUdpEndpoint`] (real WebPKI server-identity verification, no
//! insecure skip-verify), yielding a [`RustlsQuicCryptoProvider`] holding the
//! derived 1-RTT keys.
//! 2. [`AtpPacketProtection::from_provider`] adopts those handshake-derived keys
//! for the data plane (AEAD protect/unprotect + anti-replay), with no key
//! re-derivation.
//! 3. [`NativeQuicConnection`] is driven to `Established` and produces/consumes
//! application STREAM (control) + DATAGRAM (RaptorQ symbol) frames via
//! `generate_frames` / `process_packet_payload`. A [`QuicLink`] pump protects
//! each generated 1-RTT packet, sends it over UDP, and unprotects received
//! packets back into the connection.
//!
//! The reliable ATP control protocol (Hello / manifest / NeedMore / Proof /
//! Close) and the fountain feedback loop are the existing `super::` native
//! helpers; this module adds the async UDP pump plus the ATP-scoped stream PTO
//! that requeues lost control-stream offsets while the peer is waiting at a
//! protocol boundary. The in-memory loopback driver ([`super`]'s tests) moves
//! frames synchronously between two in-process connections, which cannot do real
//! async socket I/O.
//!
//! # 1-RTT wire framing (no-claim boundary)
//!
//! The handshake itself uses canonical protected long-header Initial/Handshake
//! packets (the driver's responsibility). The 1-RTT data plane here uses a
//! deliberately simplified short packet: a 9-byte clear header
//! (`0x40 | key_phase`, then the full 8-byte packet number, big-endian) used as
//! AEAD associated data, followed by `ciphertext || tag`. This mirrors the
//! handshake driver's explicit "both ends are asupersync, no QUIC header
//! protection" simplification. It is therefore **not** wire-interoperable with a
//! generic QUIC stack (no header protection, no truncated packet numbers, no
//! connection-ID demux on the short header) and is scoped to ATP-over-QUIC where
//! both peers run this code. A wire-conformant short header + header protection
//! + multi-connection demux is separate Phase-A/Phase-D work.
//!
//! # Loss posture (no-claim boundary)
//!
//! RaptorQ + the fountain feedback loop tolerate symbol-DATAGRAM loss
//! end-to-end. The reliable control STREAM has bounded, ATP-specific
//! retransmission for setup, round-complete, NeedMore, and terminal Proof
//! frames; full generic QUIC loss recovery remains out of scope for this pump.
//! Deterministic symbol loss for tests is injected before a symbol is sprayed
//! ([`QuicConfig::debug_drop_one_in`]).
use std::collections::{BTreeMap, BTreeSet, VecDeque};
use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr};
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::{Duration, Instant};
use rustls::pki_types::ServerName;
use rustls::{ClientConfig, ServerConfig};
use sha2::{Digest, Sha256};
use crate::bytes::{Bytes, BytesMut};
use crate::cx::Cx;
use crate::io::{AsyncReadExt, AsyncWriteExt};
use crate::net::atp::datagram::beacons::{BeaconMeasurement, BeaconScheduler};
use crate::net::atp::datagram::congestion::{
CongestionConfig, CongestionController, DatagramRateConfig, DatagramRateController,
DatagramRateDecision, DatagramRateSample, DatagramSendAdmission,
};
use crate::net::atp::protocol::frames::{Frame, FrameType};
use crate::net::atp::protocol::quic_frames::QuicFrame;
use crate::net::atp::quic::packet_protection::{AtpPacketProtection, AtpPacketProtectionConfig};
use crate::net::atp::transport_common::{
EntryDigest, EntryMetadata, MetadataApplyReport, flat_merkle_root_from_digests,
hash_file_streaming, hex_encode,
};
use crate::net::atp::transport_rq::erase_send;
use crate::net::quic_core::ConnectionId;
use crate::net::quic_native::handshake_driver::{
ATP_QUIC_ALPN, HandshakeLevel, QuicHandshakeDriver, client_handshake_over_udp,
is_stale_handshake_packet_error,
};
use crate::net::quic_native::tls::{
PacketProtectionRequest, PacketProtectionSpace, RustlsQuicCryptoProvider,
};
use crate::net::quic_native::{
AckRange as NativeAckRange, NativeQuicConnection, NativeQuicConnectionConfig,
NativeQuicConnectionError, OutgoingPacket, PacketNumberSpace, QuicTransportMachine,
QuicUdpEndpoint, QuicUdpEndpointConfig, ReceivedPacket, StreamId, StreamRole, StreamTableError,
};
use crate::net::{UDP_MAX_GSO_SEGMENTS, UdpSendBatchStrategy};
use crate::security::tag::TAG_SIZE;
use crate::security::{AuthenticatedSymbol, SecurityContext};
use crate::types::outcome::Outcome;
use crate::types::symbol::{Symbol, SymbolId, SymbolKind};
use super::{
NativeQuicFrameTransport, QuicBlockRepairRequest, QuicConfig, QuicControlReply,
QuicEntryEncoder, QuicHello, QuicHelloAck, QuicNeedMore, QuicPreparedSource,
QuicReceiveOptions, QuicSendLimiterReport, QuicSendStallReason, QuicSourceSymbolRequest,
QuicSprayPacingDecision, QuicStallSliceAccounting, QuicTransportError, ReceiveReceipt,
ReceiveReport, SendReport, TransferManifest, classify_blocked_flush,
classify_source_stream_admission,
};
/// Shared QUIC Initial Destination Connection ID for ATP-over-QUIC.
///
/// RFC 9001 §5.2 derives the (non-secret) Initial-space keys from the client's
/// original DCID; both peers must agree on it to derive matching Initial keys.
/// Initial keys protect only integrity against off-path tampering — the real
/// transfer confidentiality/authenticity comes from the ECDHE-derived
/// Handshake/1-RTT keys — so a fixed protocol constant here weakens nothing.
/// Using a constant (rather than peeking the first packet's header) means the
/// current accept path is single-connection-per-port; per-connection DCID demux
/// over a shared port is Phase-D work.
/// Opt-in stderr tracing for ATP/RQ benchmark diagnosis. Reuses the existing
/// ATP_RQ_TRACE switch so matrix runs can grep one trace stream across RQ and
/// QUIC transports. Defined ahead of every use: `macro_rules!` scoping is
/// textual.
macro_rules! quic_rqtrace {
($($arg:tt)*) => {
if rqtrace_enabled() {
eprintln!(
"[ATP_RQ_TRACE] [atp-quic] t={}ms {}",
rqtrace_elapsed_millis(),
format!($($arg)*)
);
}
};
}
const ATP_QUIC_INITIAL_DCID: &[u8] = &[0xA7, 0x9C, 0x10, 0xB2, 0xC3, 0xD4, 0xE5, 0xF6];
/// Client source connection ID carried in the client's handshake long headers.
const ATP_QUIC_CLIENT_SCID: &[u8] = &[0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88];
/// Server source connection ID carried in the server's handshake long headers.
const ATP_QUIC_SERVER_SCID: &[u8] = &[0x99, 0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF, 0x00];
/// Process-unique counter for QUIC receive staging directories.
static QUIC_STAGING_SEQ: AtomicU64 = AtomicU64::new(0);
/// Keep one staged output descriptor hot only for large entries where repeated
/// block-level open/seek/write/flush dominates receiver intake.
const QUIC_STAGING_FILE_CACHE_MIN_BYTES: u64 = 1024 * 1024;
/// Bound cached descriptors so tree transfers with many files do not retain one
/// file handle per entry.
const QUIC_STAGING_FILE_CACHE_MAX_ENTRIES: usize = 128;
/// Flush cached staged writes in bounded chunks. This matches the RQ staging
/// cache envelope and keeps dirty data bounded while avoiding per-block flushes.
const QUIC_STAGE_BUFFER_BYTES: usize = 256 * 1024;
fn send_native_keep_alive(
cx: &Cx,
conn: &mut NativeQuicConnection,
_control: &mut NativeQuicFrameTransport,
) -> Result<(), QuicTransportError> {
conn.queue_ping(cx)?;
Ok(())
}
async fn send_and_flush_native_keep_alive(
cx: &Cx,
link: &mut QuicLink,
control: &mut NativeQuicFrameTransport,
) -> Result<(), QuicTransportError> {
send_native_keep_alive(cx, &mut link.conn, control)?;
link.flush(cx).await?;
Ok(())
}
/// Keep-alive cadence while the receiver is busy with local work (the
/// per-member commit of a packed tree, staging writes) and would otherwise
/// neither pump nor acknowledge. The sender's proof waits are liveness-bounded,
/// so receiver silence longer than the sender's idle budget reads as a dead
/// peer even when the commit is merely slow: with the caller-driven executor
/// parked instead of self-waking (GH#67), a 2000-member commit was measured at
/// 39 s while the sender's 45 s window, started before the commit, expired
/// three seconds before the Proof existed (GH#67 follow-up, asupersync-t64ggs).
const LOCAL_WORK_KEEP_ALIVE_CADENCE: Duration = Duration::from_millis(200);
/// Rate limiter for [`keep_peer_alive_during_local_work`]: at most one
/// keep-alive per cadence, measured from the previous emission.
struct LocalWorkKeepAlive {
last: Instant,
}
impl LocalWorkKeepAlive {
fn new(now: Instant) -> Self {
Self { last: now }
}
/// True (and re-based) when `now` is at least `cadence` past the previous
/// emission; false otherwise. Never fires twice within one cadence.
fn due(&mut self, now: Instant, cadence: Duration) -> bool {
if now.saturating_duration_since(self.last) < cadence {
return false;
}
self.last = now;
true
}
}
/// Called from long local work loops on the receiver. When a keep-alive is
/// due, drain whatever the socket already holds without waiting (so the
/// sender's outstanding frames get acknowledged and its stall retransmits
/// stop), then flush: the ACK frames plus a keep-alive PING prove liveness.
/// Returns whether a keep-alive was emitted.
async fn keep_peer_alive_during_local_work(
cx: &Cx,
link: &mut QuicLink,
control: &mut NativeQuicFrameTransport,
cadence: &mut LocalWorkKeepAlive,
) -> Result<bool, QuicTransportError> {
if !cadence.due(Instant::now(), LOCAL_WORK_KEEP_ALIVE_CADENCE) {
return Ok(false);
}
emit_keep_alive(cx, link, control).await?;
Ok(true)
}
/// One keep-alive emission: drain whatever the socket already holds without
/// waiting (acknowledging the sender's outstanding frames), then flush the
/// ACKs plus a PING. Never raced against anything: a flush interrupted
/// mid-await would leave generated packets unsent and, for a queued control
/// frame, a packet number the in-flight ledger never learned about.
async fn emit_keep_alive(
cx: &Cx,
link: &mut QuicLink,
control: &mut NativeQuicFrameTransport,
) -> Result<(), QuicTransportError> {
let _ = link.pump_inbound_for(cx, Duration::ZERO).await?;
send_and_flush_native_keep_alive(cx, link, control).await
}
/// What the keep-alive loop observed when its race resolved.
enum KeepAliveTurn<T> {
/// The long step finished.
Done(Result<T, QuicTransportError>),
/// The cadence elapsed; the step is still running.
Tick,
}
/// Runs `work` (a long blocking step such as a decoded-block write or the
/// member-file batch of a packed tree, executed on a blocking thread) while
/// keeping the peer alive: at [`LOCAL_WORK_KEEP_ALIVE_CADENCE`] the socket is
/// drained and acknowledged and a keep-alive PING goes out until `work`
/// completes. Without this the receiver is silent for the whole step; a step
/// stretched by disk load past the sender's idle budget (20 s of a blocked
/// datagram cwnd, or the proof wait) made the sender declare a healthy
/// receiver dead (GH#67 follow-up, asupersync-t64ggs).
///
/// Only the cadence sleep is raced against `work`; each keep-alive emission
/// runs to completion, and `work` is always driven to completion even after
/// a keep-alive fails, because the blocking step is not cancel-safe (a
/// detached write would keep running while the staging directory is torn
/// down). A keep-alive error is reported after `work` finishes.
async fn keep_peer_alive_while<T>(
cx: &Cx,
link: &mut QuicLink,
control: &mut NativeQuicFrameTransport,
work: impl std::future::Future<Output = Result<T, QuicTransportError>>,
) -> Result<T, QuicTransportError> {
let mut work = std::pin::pin!(work);
let mut keep_alive_failure: Option<QuicTransportError> = None;
loop {
let turn = {
let step = async { KeepAliveTurn::Done(work.as_mut().await) };
let tick = async {
crate::time::sleep(cx.now(), LOCAL_WORK_KEEP_ALIVE_CADENCE).await;
KeepAliveTurn::Tick
};
futures_lite::future::or(step, tick).await
};
match turn {
KeepAliveTurn::Done(result) => {
return match keep_alive_failure {
Some(failure) => Err(failure),
None => result,
};
}
KeepAliveTurn::Tick => {
// Keep trying at every tick: a transient failure must not
// silence the receiver for the rest of the step. The last
// failure is still reported once the step has finished.
if let Err(failure) = emit_keep_alive(cx, link, control).await {
quic_rqtrace!("receiver: keep_alive_failed error={failure:?}");
keep_alive_failure = Some(failure);
}
}
}
}
}
/// RAII backstop for the native QUIC receiver staging directory.
///
/// Cooperative success and error paths remove the directory asynchronously and
/// disarm this guard. If the receive future is hard-dropped before it reaches
/// those paths, this bounded synchronous cleanup prevents partial decoded blocks
/// from leaking under the destination.
struct QuicStagingDirGuard {
dir: PathBuf,
armed: bool,
}
impl QuicStagingDirGuard {
fn new(dir: PathBuf) -> Self {
Self { dir, armed: true }
}
fn disarm(&mut self) {
self.armed = false;
}
}
impl Drop for QuicStagingDirGuard {
fn drop(&mut self) {
if self.armed {
let _ = std::fs::remove_dir_all(&self.dir);
}
}
}
async fn create_quic_staging_dir_guard(path: PathBuf) -> std::io::Result<QuicStagingDirGuard> {
crate::runtime::spawn_blocking_io(move || {
std::fs::create_dir(&path)?;
Ok(QuicStagingDirGuard::new(path))
})
.await
}
/// Bytes of the simplified 1-RTT data-plane header (flags + 8-byte packet number).
const ONE_RTT_HEADER_LEN: usize = 9;
/// QUIC AES-128-GCM authentication tag length.
const ONE_RTT_TAG_LEN: usize = 16;
/// QUIC short-header fixed bit (bit 6); set on every 1-RTT data-plane packet.
const ONE_RTT_FIXED_BIT: u8 = 0x40;
/// QUIC short-header key-phase bit (bit 2).
const ONE_RTT_KEY_PHASE_BIT: u8 = 0x04;
/// Packet-credit-sized recovery telemetry charge for one simplified ATP 1-RTT data packet.
///
/// ATP's RaptorQ repair loop, not QUIC stream retransmission, owns data
/// recovery. The native QUIC recovery machine is still useful as a packet-loss
/// signal, but its NewReno cwnd is not the data-plane admission authority:
/// erasures within the FEC budget must be handled by the fountain pacer instead
/// of a TCP-style 2x-MSS floor. Charge each symbol packet as a small virtual
/// credit so ACK/loss telemetry remains useful without throttling RaptorQ repair.
const QUIC_DATA_PLANE_TELEMETRY_PACKET_BYTES: u64 = 16;
/// Match the RQ raw-datagram pacer wait envelope: short enough for high-rate
/// clean paths, bounded enough that cancellation/liveness checkpoints keep
/// making progress under constrained lossy paths.
const QUIC_DATA_PLANE_PACER_MIN_PAUSE: Duration = Duration::from_micros(50);
const QUIC_DATA_PLANE_PACER_MAX_PAUSE: Duration = Duration::from_millis(250);
/// UDP max packet size for the link's endpoint.
///
/// The ATP-over-QUIC data plane intentionally uses large UDP datagrams on the
/// hermetic netns/veth benchmark links. Keep this as a jumbo endpoint envelope,
/// not a 1500-byte path-MTU estimate: lossy encrypted runs can coalesce an ACK or
/// small control frame with an otherwise near-full 1-RTT DATAGRAM packet, so a
/// self-imposed 16 KiB receiver cap rejected valid packets a few bytes over the
/// old bound before the fountain loop could recover.
const ATP_QUIC_UDP_MAX_PACKET: usize = 65_535;
/// UDP packet ceiling for links whose config declares real path loss.
///
/// The jumbo `ATP_QUIC_UDP_MAX_PACKET` build (8 KiB stream packets, coalesced
/// spray packets) IP-fragments on any MTU-1500 link. Fragmentation is free on
/// a clean path, but under packet loss it multiplies the effective loss rate:
/// an 8 KiB UDP datagram is ~6 fragments, so 10% per-fragment netem loss kills
/// ~47% of packets — bulk data AND the ACK/feedback path both crawl and the
/// broken regime cannot converge (br-asupersync-u6m3dy). Cap protected packets
/// under one Ethernet MTU (1350 + 28 IP/UDP = 1378 < 1500) so lossy links see
/// per-packet loss at the configured rate; clean links keep the jumbo build
/// the perfect/good-cell wins were measured with. The cap still leaves room
/// for one full default symbol DATAGRAM frame (~1200 bytes) plus packet
/// overhead; `udp_packet_cap_for_config` raises it further if an operator
/// configures oversized symbols, because a payload budget below one symbol
/// frame fail-closes the spray path at startup.
const QUIC_LOSSY_PATH_UDP_MAX_PACKET: usize = 1_350;
/// Select the protected-packet build ceiling from the declared loss posture.
fn udp_packet_cap_for_config(config: &QuicConfig) -> usize {
if config.round0_loss_target >= super::QUIC_FEEDBACK_REPAIR_LOSS_ENABLE_MIN {
// Floor: one symbol DATAGRAM frame (either auth posture) or one
// configured DATAGRAM must always fit a packet, else spray-path
// budgets invert (`spray_frame_payload_limit` clamps min > max).
let symbol_frame =
symbol_datagram_frame_len(config.symbol_size, super::AUTH_ENVELOPE_HEADER_LEN)
.max(symbol_datagram_frame_len(
config.symbol_size,
super::ENVELOPE_HEADER_LEN,
))
.max(config.max_datagram_size);
QUIC_LOSSY_PATH_UDP_MAX_PACKET.max(
symbol_frame
.saturating_add(ONE_RTT_PACKET_OVERHEAD)
.saturating_add(ONE_RTT_COALESCED_CONTROL_HEADROOM),
)
} else {
ATP_QUIC_UDP_MAX_PACKET
}
}
/// Bytes added around each encoded 1-RTT payload before it is handed to UDP.
const ONE_RTT_PACKET_OVERHEAD: usize = ONE_RTT_HEADER_LEN + ONE_RTT_TAG_LEN;
/// Reserved bytes below the endpoint cap for ACK/control-frame coalescing slack.
const ONE_RTT_COALESCED_CONTROL_HEADROOM: usize = 64;
/// Loss-free encrypted sprays batch several full protected packets per UDP
/// send so Linux UDP_SEGMENT can amortize syscall cost above the packet-fill
/// layer. Lossy paths keep the pacing burst unchanged.
const QUIC_CLEAN_GSO_PACKETS_PER_FLUSH: usize = 4;
/// Keep the clean GSO spray batch below `NativeQuicConnection`'s bounded
/// outbound DATAGRAM queue (currently 256) so batching never drops queued
/// symbols before `flush()` drains them.
const QUIC_CLEAN_GSO_MAX_FLUSH_SYMBOLS: usize = 255;
/// Minimum clean-link spray burst in symbols (MATRIX-108 / bead 839ykg).
///
/// The RTT-derived `max_burst_symbols` collapses to ~2 on a low-latency
/// encrypted path (one symbol per protected packet ⇒ `packet_width≈1` ⇒ a
/// `packet_width × QUIC_CLEAN_GSO_PACKETS_PER_FLUSH` floor of only ~4), so the
/// send flushes tiny bursts and pays per-flush QUIC packet-protection cost on
/// every couple of symbols — capping the encrypted send at ~5 MB/s instead of
/// its ~24 MiB/s budget. Match the rq path's 16–32 symbol burst
/// (`RQ_ADAPTIVE_BURST_SYMBOLS`) on clean links so a flush amortizes that work;
/// the post-burst byte-paced sleep keeps the average rate at the budget.
const QUIC_CLEAN_SPRAY_BURST_FLOOR_SYMBOLS: usize = 32;
/// Loss ceiling below which a spray uses the clean coalescing/burst-floor path
/// (amortize per-flush QUIC packet-protection over a 32-symbol burst) instead of
/// the per-symbol lossy path. A strict `<= f64::EPSILON` gate left the encrypted
/// near-clean path — which measures a trace of startup/handshake loss — in the
/// per-symbol branch, so the QUIC_CLEAN_SPRAY_BURST_FLOOR_SYMBOLS floor (008e9c7e1)
/// never engaged and the send stayed at ~2 symbols/flush, ~5 MB/s (MATRIX-109).
/// `bad` (2%) and `broken` (10%) stay above this ceiling and keep per-symbol
/// pacing to preserve loss granularity on constrained links.
const QUIC_CLEAN_SPRAY_MAX_LOSS_RATE: f64 = 0.01;
/// Clean jumbo coalescing is for low-loss, low-RTT LAN-style paths. On the
/// 50M/bad encrypted cell the handshake RTT is ~80ms; treating that as clean
/// packed 54 symbols into one ~63 KiB UDP datagram, so one shaped-link packet
/// loss erased a whole symbol group and stalled recovery behind cwnd.
const QUIC_CLEAN_COALESCING_MAX_RTT_S: f64 = 0.050;
/// Keep native ATP-QUIC symbol packets loss-granular until the lossy convergence
/// gate is banked. The current recovery RTT is a synthetic pump clock sample,
/// not a trustworthy wall-clock path RTT, so an RTT-based clean gate can still
/// misclassify the 50M/bad cell as jumbo-safe.
const QUIC_NATIVE_CLEAN_JUMBO_COALESCING_ENABLED: bool = false;
/// Upper bound for symbols handed from the RQ producer loop to the QUIC sender
/// pump in one in-process turn on lossy paths. Clean paths may hand off the
/// whole bounded flush window so they do not split one GSO-ready burst into
/// multiple scheduler-visible producer/sender turns.
const QUIC_LOSSY_SPRAY_HANDOFF_MAX_SYMBOLS: usize = 64;
/// Fixed socket buffer budget for the native ATP-QUIC link. This is intentionally
/// a constant envelope, not proportional to object size, so large transfers cannot
/// force process/object-sized buffering while loopback proof runs avoid kernel
/// receive-buffer drops during one-round RaptorQ sprays.
const ATP_QUIC_UDP_SOCKET_BUFFER: usize = 16 * 1024 * 1024;
/// Packets pulled from the socket per inbound pump. Each received UDP packet is
/// copied through packet protection, frame decode, and the application DATAGRAM
/// queue before the symbol decoder can consume it, so the batch width is part of
/// the native link's memory envelope. Full batches trigger a bounded quiet-drain
/// loop below so large bursts are drained before the receiver waits on control.
const INBOUND_PUMP_BATCH: usize = 512;
/// Maximum full receive batches drained in one pump turn. This preserves the
/// F1.1 drain-until-empty behavior for ordinary bursts while bounding a single
/// turn under sustained peer flooding.
const INBOUND_PUMP_MAX_DRAIN_BATCHES: usize = 64;
/// Receiver symbol batches fed before giving the UDP socket another immediate
/// drain chance. Keeping this at one prevents a full userspace DATAGRAM queue
/// from monopolizing the receive task while the kernel socket buffer is filling.
const RECEIVER_SYMBOL_DRAIN_BATCHES_PER_SOCKET_POLL: usize = 1;
/// After the first full batch, wait only a tiny quiet window for the next batch.
/// `UdpSocket::recv_batch_from` drains immediately-ready datagrams internally;
/// this grace covers the full-batch case where the kernel may still have more
/// packets queued without charging a full idle timeout to every drain attempt.
const INBOUND_PUMP_DRAIN_GRACE: Duration = Duration::from_millis(1);
/// Headroom for 1-RTT short-header, AEAD tag, and STREAM frame varints when
/// sizing recovery-governed STREAM packets. `NativeQuicConnection` already
/// reserves 32 frame bytes internally, so this must stay below the small cwnd
/// tail observed at the 2400-byte floor.
const QUIC_STREAM_PACKET_OVERHEAD_BUDGET: u64 = 48;
/// Bytes of source STREAM payload queued before giving the socket pump a turn.
/// GOOD source-stream sends stay inside the native QUIC packet envelope: larger
/// packets amplified loss tails on shaped netns links, while this envelope keeps
/// the repair tail small and avoids IP fragmentation.
const QUIC_SOURCE_STREAM_FLUSH_BYTES: u64 = 512 * 1024;
/// Upper bound on un-flushed send-side stream bytes; the disk reader waits on
/// paced flushes past this, so sender RSS stays a few bursts deep instead of
/// scaling with the transfer size.
const QUIC_SOURCE_STREAM_SEND_QUEUE_MAX_BYTES: u64 = 8 * 1024 * 1024;
/// Upper bound on sent-but-unacknowledged source-stream bytes. This is the
/// runaway guard, not a congestion window: it sits far above any bench-link
/// BDP (1 gbit × 25 ms ≈ 3 MB) so it never limits clean throughput, but it
/// stops a mis-estimated pacing rate from streaming hundreds of MB into a
/// loss gap (which drove the receiver's reassembly backlog quadratic and
/// timed out the 500M A/B). New data admission waits on the ACK clock past
/// this; retransmits are unaffected.
const QUIC_SOURCE_STREAM_UNACKED_MAX_BYTES: u64 = 16 * 1024 * 1024;
// HISTORY (uw1cc2 / MATRIX-225→226): a BDP in-flight cap fed by ACK-window
// aggregates was refuted twice — (a) the transport RTT estimator runs on
// this path's synthetic app-data clock and read ~1 ms on a 50 ms link,
// flooring the cap into an ACK-clock stall; (b) an offered-rate clamp on
// window aggregates MANUFACTURED capacity evidence and collapsed the cell
// (155 s, 1101 MB re-sent). The cap below is the MATRIX-226 rebuild on
// honest inputs: per-packet delivered-counter samples
// (SourceStreamDeliverySampler) for BtlBw and wall-clock send→ack minima
// for RTprop, validated in the deterministic lab gate
// (matrix226_delivery_lab tests) before any bench run.
/// BDP multiple for the source stream's delivery-clocked in-flight cap
/// (`cwnd = gain × BtlBw × RTprop`, uw1cc2 SPEC item 3): headroom for one
/// probe cycle above the pipe, per BBR. Deliberately NOT loss-reactive —
/// NewReno-style loss-halving on this path measured a 12× mild-loss
/// regression (MATRIX-202) and stays refuted; random loss leaves the
/// delivery estimate (and so this cap) untouched, while genuine congestion
/// shrinks measured delivery and pulls the cap down with it.
const QUIC_SOURCE_STREAM_BDP_CWND_GAIN_X1000: u64 = 2000;
/// Floor for the BDP in-flight cap: one bounded receive window, so the cap
/// never binds tighter than the flow-control window already does. Tracks
/// [`QUIC_SOURCE_STREAM_RECV_WINDOW_BYTES`] — the cap formula uses
/// RTprop_min (pure flight time) while `stream_unacked_bytes` accounting
/// includes ACK-processing lag, so the computed cap under-sizes by ~1.6×
/// on ACK-batchy paths and the floor is what actually holds the invariant.
const QUIC_SOURCE_STREAM_BDP_CWND_MIN_BYTES: u64 = QUIC_SOURCE_STREAM_RECV_WINDOW_BYTES;
/// Delivery-clocked in-flight cap for the reliable source stream:
/// `clamp(gain × BtlBw × RTprop, window-floor, runaway-ceiling)`.
///
/// Until an RTprop sample exists (or while the delivery filter is empty)
/// the cap falls back to the 16 MiB runaway ceiling — admission then
/// behaves exactly as before the cap existed.
fn source_stream_bdp_admission_cap(bottleneck_bytes_per_s: u64, rtprop_micros: Option<u64>) -> u64 {
let Some(rtt_micros) = rtprop_micros.filter(|micros| *micros > 0) else {
return QUIC_SOURCE_STREAM_UNACKED_MAX_BYTES;
};
if bottleneck_bytes_per_s == 0 {
return QUIC_SOURCE_STREAM_UNACKED_MAX_BYTES;
}
let bdp =
u128::from(bottleneck_bytes_per_s).saturating_mul(u128::from(rtt_micros)) / 1_000_000u128;
let cwnd = bdp.saturating_mul(u128::from(QUIC_SOURCE_STREAM_BDP_CWND_GAIN_X1000)) / 1000u128;
u64::try_from(cwnd).unwrap_or(u64::MAX).clamp(
QUIC_SOURCE_STREAM_BDP_CWND_MIN_BYTES,
QUIC_SOURCE_STREAM_UNACKED_MAX_BYTES,
)
}
/// Bounded receive window for the paced source stream, negotiated in the
/// HelloAck: the receiver advertises `read_offset + window` via
/// MAX_STREAM_DATA and a compliant sender installs the window as its initial
/// send-credit limit. Bounds receiver reassembly RSS to roughly one window
/// as a side effect.
///
/// SIZING IS A PATH LAW, NOT A TUNABLE (MATRIX-228, measured both ways):
/// the good-regime shaper queue (~1.4 MB) is SMALLER than the lag-inflated
/// effective BDP (25 MB/s × ~82 ms = 50 ms RTT + ~30 ms receiver ACK
/// cadence ≈ 2.05 MB). Any in-flight bound at or below BDP_eff drains the
/// pipe on every head-of-line repair (2 MiB measured: unacked pinned at
/// the edge, 64 % duty cycle, 500M/good 47.0 s); any bound above it
/// overfills the queue (4 MiB measured: delivery improved — samples
/// 16.3→20.5 MB/s — but repair volume exploded 39→289 MB and walls went
/// 47.0→66 s uniform). 2 MiB sits at the optimum of that trade-off for
/// every bench regime. CORRECTION (MATRIX-229, measured): the ~30 ms lag
/// term is NOT ACK cadence — the receiver flushes ACK/credit packets
/// every ~1.3 ms (34.7 K over a 44 s rep) — it is REPAIR-EPISODE SMEARING:
/// each of ~650 hole repairs delays its bytes' acknowledgement by
/// detect+resend+RTT (~55-105 ms), inflating the average flight to ~82 ms
/// against a 49 ms RTprop. With repair already threshold-fast (zero PTO
/// events) and hole count fixed by per-packet loss, the ~47 s wall is the
/// architectural ceiling of an un-FEC'd in-order stream on a
/// queue-smaller-than-BDP path. Earlier history: gate31-33 found the same
/// 2 MiB optimum empirically under the constant-gain pacer. Env override:
/// `ATP_QUIC_STREAM_RECV_WINDOW` (bytes, min 64 KiB).
///
/// This is the STARTING window. On a clean high-RTT path it is the binding
/// limiter (A3, 2026-09-03: `wan` 90 ms × 300 mbit, BDP 3.4 MB, the sender
/// sat on `MAX_STREAM_DATA` credit 90 % of the time at half the link), so
/// the receiver lets it grow — but only when the sender asks with
/// STREAM_DATA_BLOCKED, which the sender does only while credit-bound AND
/// loss-clean (see `wait_source_stream_send_admission`). A queue-limited or
/// lossy path never asks, so it keeps the 2 MiB path law above; measured
/// with static windows before the gate existed: 8 MiB took `good` from
/// 43 s to 68 s (61 % of the file re-sent) and killed `bad` outright, while
/// 4 MiB and 8 MiB both took `wan` from 25.9 s to 15.7 s.
const QUIC_SOURCE_STREAM_RECV_WINDOW_BYTES: u64 = 2 * 1024 * 1024;
/// Ceiling for sender-gated growth of the source-stream receive window
/// (`ATP_QUIC_STREAM_RECV_WINDOW_MAX` overrides; a value at or below the
/// starting window disables growth). 4 MiB captured the whole `wan` win
/// (15.66 s, identical to 8 MiB) at lower receiver RSS and stays under the
/// reassembly fragment guard for full-size stream frames; the receiver also
/// clamps the cap by that guard for its own MTU
/// (`quic_source_stream_recv_window_growth_cap`). Growing past it needs
/// contiguous reassembly chunks to coalesce first (8 MiB behind one hole
/// buffers ~7000 chunks against the 4096 guard — the `bad` failure).
const QUIC_SOURCE_STREAM_RECV_WINDOW_MAX_BYTES: u64 = 4 * 1024 * 1024;
fn quic_source_stream_recv_window_bytes() -> u64 {
std::env::var("ATP_QUIC_STREAM_RECV_WINDOW")
.ok()
.and_then(|value| value.trim().parse::<u64>().ok())
.filter(|window| *window >= 64 * 1024)
.unwrap_or(QUIC_SOURCE_STREAM_RECV_WINDOW_BYTES)
}
fn quic_source_stream_recv_window_max_bytes() -> u64 {
std::env::var("ATP_QUIC_STREAM_RECV_WINDOW_MAX")
.ok()
.and_then(|value| value.trim().parse::<u64>().ok())
.unwrap_or(QUIC_SOURCE_STREAM_RECV_WINDOW_MAX_BYTES)
}
/// The receive-window growth ceiling the receiver may safely allow: the
/// configured maximum, clamped so a window full of minimum-size stream
/// frames behind a single hole stays under the reassembly fragment guard
/// with headroom (`None` when growth is disabled or nothing fits).
fn quic_source_stream_recv_window_growth_cap(
window: u64,
configured_max: u64,
max_app_payload: usize,
) -> Option<u64> {
let fragment_safe =
(crate::net::quic_native::connection::MAX_BUFFERED_STREAM_REASSEMBLY_FRAGMENTS as u64)
.saturating_sub(64)
.saturating_mul(u64::try_from(max_app_payload).unwrap_or(0));
let cap = configured_max.min(fragment_safe);
(cap > window).then_some(cap)
}
/// Inbound drain budget (batches) for the source-stream read path. The read
/// loop drains the reassembly map between pumps, so a small budget bounds
/// receiver-side backlog RSS; the FEC DATAGRAM pump keeps the larger
/// `INBOUND_PUMP_MAX_DRAIN_BATCHES` drain-until-empty behavior.
///
/// This is also the receiver's ACK cadence: the read loop flushes ACKs once
/// per pump turn, so one turn must stay small (one batch ≈ 4 MB) — at 4
/// batches the ~16 MB ACK batches lockstepped with the sender's un-ACKed
/// ceiling into a hard ~64 MB/s throughput cap.
const QUIC_SOURCE_STREAM_READ_DRAIN_BATCHES: usize = 1;
const QUIC_SOURCE_STREAM_PACKET_BYTES: usize = 8 * 1024;
const QUIC_SOURCE_STREAM_REPAIR_LOSS_MULTIPLIER: f64 = 4.0;
fn source_stream_max_frame_bytes() -> usize {
one_rtt_max_payload_for_udp_packet(QUIC_SOURCE_STREAM_PACKET_BYTES).max(1)
}
fn source_stream_pacing_interval(frame_bytes: usize, pacing_rate_bps: u64) -> Duration {
let bytes = u128::try_from(frame_bytes.max(1)).unwrap_or(u128::MAX);
let rate = u128::from(pacing_rate_bps.max(1));
let nanos = bytes
.saturating_mul(1_000_000_000)
.saturating_add(rate.saturating_sub(1))
/ rate;
Duration::from_nanos(u64::try_from(nanos.max(1)).unwrap_or(u64::MAX))
}
/// Control-plane PTO. When the receiver is awaiting repair after a NeedMore and the link goes idle,
/// the NeedMore (receiver->sender) or the repair round (sender->receiver) was likely lost on the wire
/// — ATP control rides best-effort 1-RTT here, so under real-internet loss a single dropped NeedMore
/// otherwise deadlocks both sides until the full idle timeout. Re-send the NeedMore on this interval
/// instead; this is what lets cross-machine transfers converge through control-frame loss.
const NEEDMORE_PTO: Duration = Duration::from_millis(1500);
/// Fast retry cadence for source STREAM recovery while bytes are still being
/// flushed or while the sender awaits the receiver's reliable Proof.
const SOURCE_STREAM_PTO: Duration = Duration::from_millis(200);
/// Absolute bound on a proof/feedback wait, as a multiple of the idle
/// timeout. The wait's silence budget restarts on every 1-RTT packet so a
/// receiver that is alive but slow (a long local commit under keep-alives)
/// is not declared dead; this cap keeps a peer that is alive at the QUIC
/// layer yet never delivers the frame from holding the session, its file
/// handles and its retained in-flight frames forever.
const PROOF_WAIT_LIVENESS_MULTIPLIER: u32 = 8;
const QUIC_LOSS_TARGET_PROGRESS_STALL_RATIO: f64 = 0.50;
const QUIC_LOSS_TARGET_PROGRESS_LOSS_MARGIN: f64 = 0.01;
const QUIC_LOSS_TARGET_DELIVERY_BACKOFF_HEADROOM: f64 = 1.25;
/// Conservative RTT used before ACK samples establish the real lossy-path RTprop.
const QUIC_LOSSY_COLD_START_RTT_MICROS: u64 = 200_000;
const QUIC_STILL_VIABLE_FEEDBACK_GRACE_ROUNDS: u32 = 8;
const REPAIR_ROUND_RECEIVE_GRACE_BYTES_PER_S: u64 = 128 * 1024;
/// Bounded terminal Proof retransmits. This uses STREAM-offset requeue, not a
/// duplicate higher-offset Proof, so it fills the receiver->sender stream gap
/// that otherwise leaves the sender waiting until its idle timeout.
const TERMINAL_PROOF_RETRANSMIT_ATTEMPTS: u32 = 4;
/// Quiet window that ends a receiver symbol round after at least one symbol was
/// accepted without seeing ObjectComplete. It intentionally matches the
/// control-plane PTO: encrypted/coalesced sprays can have short paced gaps, and
/// cutting a round sooner makes the receiver emit stale zero-`round_symbols_sent`
/// NeedMore frames before the sender has actually completed the spray.
const ROUND_PROGRESS_IDLE_GRACE: Duration = NEEDMORE_PTO;
/// Minimum NeedMore re-sends while awaiting one round's repair before giving up.
///
/// The live budget is derived from [`QuicConfig::idle_timeout`], so explicit
/// short-timeout tests still fail fast while the default encrypted lossy lane
/// gets a larger convergence window.
const MIN_NEEDMORE_PTO_ATTEMPTS: u32 = 1;
/// Idle-budget accounting for `next_control_frame_with_stream_pto`: a pump
/// window that delivered any 1-RTT packet (a keep-alive PING during the peer's
/// long local commit included) proves the peer is alive and restarts the
/// count, so the budget bounds CONSECUTIVE silent windows exactly like
/// `next_control_frame`'s idle window and the source-stream recovery clock.
/// Counting silent windows cumulatively let a receiver that was alive but slow
/// (its commit crawls under the parked caller-driven executor, GH#67
/// follow-up) exhaust the sender's budget between keep-alives.
fn stream_pto_attempts_after_pump(attempts: u32, pumped: usize) -> u32 {
if pumped > 0 {
0
} else {
attempts.saturating_add(1)
}
}
fn needmore_pto_attempt_budget(idle_timeout: Duration) -> u32 {
let pto_millis = NEEDMORE_PTO.as_millis().max(1);
let idle_millis = idle_timeout.as_millis().max(pto_millis);
let attempts = idle_millis.div_ceil(pto_millis);
u32::try_from(attempts)
.unwrap_or(u32::MAX)
.max(MIN_NEEDMORE_PTO_ATTEMPTS)
}
fn duration_for_paced_bytes(bytes: u64, bytes_per_second: u64) -> Duration {
let nanos = u128::from(bytes)
.saturating_mul(1_000_000_000)
.div_ceil(u128::from(bytes_per_second.max(1)));
Duration::from_nanos(u64::try_from(nanos.max(1)).unwrap_or(u64::MAX))
}
fn paced_repair_round_idle_grace(
config: &QuicConfig,
need: Option<&QuicNeedMore>,
symbol_datagram_frame_len: usize,
) -> Duration {
let Some(need) = need else {
return ROUND_PROGRESS_IDLE_GRACE;
};
let requested_symbols = need_more_requested_symbol_count(need);
if requested_symbols == 0 {
return ROUND_PROGRESS_IDLE_GRACE;
}
let pacing_bytes_per_second =
super::quic_repair_pacing_cap_bps(config, need.round_loss_fraction)
.unwrap_or(super::QUIC_DEFAULT_COLD_START_PACING_BYTES_PER_S as u64)
.min(REPAIR_ROUND_RECEIVE_GRACE_BYTES_PER_S)
.max(1);
let frame_bytes = u64::try_from(symbol_datagram_frame_len.max(1)).unwrap_or(u64::MAX);
let requested_bytes = requested_symbols.saturating_mul(frame_bytes);
duration_for_paced_bytes(requested_bytes, pacing_bytes_per_second)
.saturating_add(NEEDMORE_PTO)
.clamp(ROUND_PROGRESS_IDLE_GRACE, config.idle_timeout)
}
/// Milliseconds since the first trace line of this process, so interleaved
/// sender/receiver traces can be ordered on one clock.
fn rqtrace_elapsed_millis() -> u128 {
static START: std::sync::OnceLock<Instant> = std::sync::OnceLock::new();
START.get_or_init(Instant::now).elapsed().as_millis()
}
/// Under `ATP_RQ_TRACE`, time three trivial `spawn_blocking_io` round trips
/// and trace each. A driverless caller whose blocking completions only get
/// noticed on somebody else's wake shows up here as multi-second hops; a
/// healthy wake path completes each hop in microseconds (GH#67 follow-up).
async fn trace_blocking_roundtrip_probe(label: &'static str) {
if !rqtrace_enabled() {
return;
}
for hop in 0..3u8 {
let before = Instant::now();
let closure_started =
crate::runtime::spawn_blocking_io(|| Ok::<Instant, std::io::Error>(Instant::now()))
.await
.unwrap_or(before);
let resumed = Instant::now();
quic_rqtrace!(
"blocking_roundtrip_probe label={label} hop={hop} spawn_us={} wake_us={} total_us={}",
closure_started
.saturating_duration_since(before)
.as_micros(),
resumed
.saturating_duration_since(closure_started)
.as_micros(),
resumed.saturating_duration_since(before).as_micros(),
);
}
}
/// True when `ATP_RQ_TRACE` opt-in tracing is enabled for this process.
/// Whether `ATP_RQ_TRACE` tracing is on. Read once per process: the check
/// sits on every socket wait, and an environment lookup there takes the
/// process-global environment lock each time.
fn rqtrace_enabled() -> bool {
static ENABLED: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
*ENABLED.get_or_init(|| std::env::var_os("ATP_RQ_TRACE").is_some())
}
/// Snapshot of the process-global fallback I/O driver counters (GH#67
/// follow-up). Only `test-internals` builds carry the counters; elsewhere the
/// snapshot is always empty and the diagnostics say so.
#[cfg(all(not(target_arch = "wasm32"), any(test, feature = "test-internals")))]
type FallbackIoProbe = Option<crate::net::udp::FallbackIoDriverProbe>;
#[cfg(not(all(not(target_arch = "wasm32"), any(test, feature = "test-internals"))))]
type FallbackIoProbe = Option<()>;
#[cfg(all(not(target_arch = "wasm32"), any(test, feature = "test-internals")))]
fn fallback_io_probe_now() -> FallbackIoProbe {
crate::net::udp::fallback_io_driver_probe()
}
#[cfg(not(all(not(target_arch = "wasm32"), any(test, feature = "test-internals"))))]
fn fallback_io_probe_now() -> FallbackIoProbe {
None
}
/// Probe snapshot taken only when tracing is on, so the untraced hot path
/// never locks the fallback driver.
fn rqtrace_fallback_io_probe() -> FallbackIoProbe {
if rqtrace_enabled() {
fallback_io_probe_now()
} else {
None
}
}
#[cfg(all(not(target_arch = "wasm32"), any(test, feature = "test-internals")))]
fn describe_fallback_io_probe(now: FallbackIoProbe) -> String {
let Some(now) = now else {
return "fallback_io=unstarted".to_string();
};
format!(
"fallback_io_polls={} fallback_io_events={} fallback_io_wakers={} \
fallback_io_unknown={} fallback_io_regs={} fallback_io_deregs={} \
fallback_sock_regs={} fallback_rearms={} fallback_migrations={} \
fallback_self_wakes={}",
now.polls,
now.events_received,
now.wakers_dispatched,
now.unknown_tokens,
now.registrations,
now.deregistrations,
now.fallback_registrations,
now.fallback_rearms,
now.fallback_migrations,
now.fallback_self_wakes,
)
}
#[cfg(not(all(not(target_arch = "wasm32"), any(test, feature = "test-internals"))))]
fn describe_fallback_io_probe(_now: FallbackIoProbe) -> String {
"fallback_io=unavailable".to_string()
}
/// Fallback driver activity between two snapshots (what the pump did while
/// the caller was parked).
#[cfg(all(not(target_arch = "wasm32"), any(test, feature = "test-internals")))]
fn describe_fallback_io_delta(now: FallbackIoProbe, before: FallbackIoProbe) -> String {
let (Some(now), Some(before)) = (now, before) else {
return "fallback_io_delta=unavailable".to_string();
};
format!(
"d_polls={} d_events={} d_wakers={} d_rearms={}",
now.polls.saturating_sub(before.polls),
now.events_received.saturating_sub(before.events_received),
now.wakers_dispatched
.saturating_sub(before.wakers_dispatched),
now.fallback_rearms.saturating_sub(before.fallback_rearms),
)
}
#[cfg(not(all(not(target_arch = "wasm32"), any(test, feature = "test-internals"))))]
fn describe_fallback_io_delta(_now: FallbackIoProbe, _before: FallbackIoProbe) -> String {
"fallback_io_delta=unavailable".to_string()
}
/// Socket waits shorter than this are drain-grace polls between batches,
/// not idle parks; `pump_idle` traces only real parks.
const PUMP_IDLE_TRACE_MIN_WAIT: Duration = Duration::from_millis(100);
/// Opt-in cached-staging cursor audit shared with the RQ transport. When
/// `ATP_RQ_STAGING_CURSOR_AUDIT` is set, a QUIC write that would otherwise
/// trust the tracked cursor verifies the file's actual cursor first and
/// self-heals any mismatch. Off by default because the check adds one seek
/// query per sequential cached write.
fn quic_staging_cursor_audit_enabled() -> bool {
static ENABLED: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
*ENABLED.get_or_init(|| std::env::var_os("ATP_RQ_STAGING_CURSOR_AUDIT").is_some())
}
fn trace_quic_flush_coalescing(
cx: &Cx,
packets: usize,
datagram_frames: usize,
observed_max_datagram_frames_per_packet: usize,
configured_max_symbol_frames_per_packet: usize,
plaintext_payload_bytes: usize,
protected_udp_bytes: usize,
native_send_batch_used: bool,
gso_send_used: bool,
fallback_used: bool,
) {
if packets == 0 || std::env::var_os("ATP_RQ_TRACE").is_none() {
return;
}
let avg_datagram_frames_per_packet_x100 = datagram_frames.saturating_mul(100) / packets.max(1);
let packets_s = packets.to_string();
let datagram_frames_s = datagram_frames.to_string();
let observed_max_datagram_frames_per_packet_s =
observed_max_datagram_frames_per_packet.to_string();
let configured_max_symbol_frames_per_packet_s =
configured_max_symbol_frames_per_packet.to_string();
let avg_datagram_frames_per_packet_x100_s = avg_datagram_frames_per_packet_x100.to_string();
let plaintext_payload_bytes_s = plaintext_payload_bytes.to_string();
let protected_udp_bytes_s = protected_udp_bytes.to_string();
let native_send_batch_used_s = native_send_batch_used.to_string();
let gso_send_used_s = gso_send_used.to_string();
let fallback_used_s = fallback_used.to_string();
cx.trace_with_fields(
"atp_quic.sender.flush_coalescing",
&[
("packets", packets_s.as_str()),
("datagram_frames", datagram_frames_s.as_str()),
(
"observed_max_datagram_frames_per_packet",
observed_max_datagram_frames_per_packet_s.as_str(),
),
(
"configured_max_symbol_frames_per_packet",
configured_max_symbol_frames_per_packet_s.as_str(),
),
(
"avg_datagram_frames_per_packet_x100",
avg_datagram_frames_per_packet_x100_s.as_str(),
),
(
"plaintext_payload_bytes",
plaintext_payload_bytes_s.as_str(),
),
("protected_udp_bytes", protected_udp_bytes_s.as_str()),
("native_send_batch_used", native_send_batch_used_s.as_str()),
("gso_send_used", gso_send_used_s.as_str()),
("fallback_used", fallback_used_s.as_str()),
],
);
quic_rqtrace!(
"sender: flush_coalescing packets={} datagram_frames={} observed_max_datagrams_per_packet={} configured_max_symbol_frames_per_packet={} avg_datagrams_per_packet_x100={} plaintext_payload_bytes={} protected_udp_bytes={} native_send_batch_used={} gso_send_used={} fallback_used={}",
packets,
datagram_frames,
observed_max_datagram_frames_per_packet,
configured_max_symbol_frames_per_packet,
avg_datagram_frames_per_packet_x100,
plaintext_payload_bytes,
protected_udp_bytes,
native_send_batch_used,
gso_send_used,
fallback_used,
);
}
fn trace_quic_symbol_handoff(
cx: &Cx,
symbols: usize,
encoded_bytes: usize,
queue_before: usize,
queue_after: usize,
flush_symbol_limit: usize,
flushed_packets: usize,
pacing_rate_bps: u64,
) {
if symbols == 0 || std::env::var_os("ATP_RQ_TRACE").is_none() {
return;
}
let symbols_s = symbols.to_string();
let encoded_bytes_s = encoded_bytes.to_string();
let queue_before_s = queue_before.to_string();
let queue_after_s = queue_after.to_string();
let flush_symbol_limit_s = flush_symbol_limit.to_string();
let flushed_packets_s = flushed_packets.to_string();
let pacing_rate_bps_s = pacing_rate_bps.to_string();
cx.trace_with_fields(
"atp_quic.sender.symbol_handoff",
&[
("symbols", symbols_s.as_str()),
("encoded_bytes", encoded_bytes_s.as_str()),
("queue_before", queue_before_s.as_str()),
("queue_after", queue_after_s.as_str()),
("flush_symbol_limit", flush_symbol_limit_s.as_str()),
("flushed_packets", flushed_packets_s.as_str()),
("pacing_rate_bps", pacing_rate_bps_s.as_str()),
],
);
quic_rqtrace!(
"sender: symbol_handoff symbols={} encoded_bytes={} queue_before={} queue_after={} flush_symbol_limit={} flushed_packets={} pacing_rate_bps={}",
symbols,
encoded_bytes,
queue_before,
queue_after,
flush_symbol_limit,
flushed_packets,
pacing_rate_bps,
);
}
fn need_more_repair_symbol_count(need: &QuicNeedMore) -> u64 {
need.repair_blocks.iter().fold(0u64, |acc, request| {
acc.saturating_add(u64::from(request.symbols))
})
}
fn need_more_requested_symbol_count(need: &QuicNeedMore) -> u64 {
need_more_repair_symbol_count(need)
.saturating_add(u64::try_from(need.source_symbols.len()).unwrap_or(u64::MAX))
}
fn infer_missing_round_complete_symbols(
expected_round: u32,
observed_symbols: u64,
last_need: Option<&QuicNeedMore>,
) -> u64 {
last_need
.filter(|need| need.feedback_round == expected_round)
.map(need_more_requested_symbol_count)
.unwrap_or(observed_symbols)
.max(observed_symbols)
}
fn trace_inferred_round_complete_symbols(
cx: &Cx,
expected_round: u32,
observed_symbols: u64,
inferred_symbols: u64,
) {
let expected_round_s = expected_round.to_string();
let observed_symbols_s = observed_symbols.to_string();
let inferred_symbols_s = inferred_symbols.to_string();
cx.trace_with_fields(
"atp_quic.receive.inferred_round_complete_symbols",
&[
("round", expected_round_s.as_str()),
("round_symbols_observed", observed_symbols_s.as_str()),
("round_symbols_sent", inferred_symbols_s.as_str()),
],
);
quic_rqtrace!(
"receiver: inferred ObjectComplete round={} round_symbols_observed={} round_symbols_sent={}",
expected_round,
observed_symbols,
inferred_symbols,
);
super::quic_progress(format_args!(
"receiver: object_complete_inferred round={expected_round} observed={observed_symbols} round_symbols_sent={inferred_symbols}"
));
}
fn emit_receiver_round_block_symbols(
transfer_id: &str,
round: u32,
block_symbols: &BTreeMap<(u32, u8), (u64, u64)>,
) {
for ((entry, sbn), (observed, accepted)) in block_symbols {
super::quic_progress(format_args!(
"receiver: block_symbols round={round} transfer={transfer_id} entry={entry} sbn={sbn} observed={observed} accepted={accepted}"
));
}
}
#[allow(
clippy::cast_possible_truncation,
clippy::cast_precision_loss,
clippy::cast_sign_loss
)]
fn loss_compensated_repair_target(base_deficit_symbols: u64, loss_fraction: Option<f64>) -> u64 {
if base_deficit_symbols == 0 {
return 0;
}
let Some(loss) = loss_fraction.filter(|loss| loss.is_finite()) else {
return base_deficit_symbols;
};
if loss <= 0.0 {
return base_deficit_symbols;
}
let effective_loss = loss
.max(super::QUIC_FEEDBACK_REPAIR_LOSS_ENABLE_MIN)
.min(super::QUIC_FEEDBACK_REPAIR_MAX_OVERHEAD);
let compensated_loss = (effective_loss
* (1.0 + super::QUIC_FEEDBACK_REPAIR_LOSS_MARGIN_FRACTION)
+ super::QUIC_FEEDBACK_REPAIR_LOSS_MARGIN_MIN)
.clamp(0.0, 0.90);
let delivery_fraction = (1.0 - compensated_loss).max(0.10);
((base_deficit_symbols as f64) / delivery_fraction).ceil() as u64
}
fn need_more_base_deficit_symbols(need: &QuicNeedMore, requested_repair_symbols: u64) -> u64 {
need.repair_base_deficit_symbols
.unwrap_or(requested_repair_symbols)
}
fn need_more_loss_compensated_target_symbols(
need: &QuicNeedMore,
base_deficit_symbols: u64,
) -> u64 {
need.repair_loss_compensated_target_symbols
.unwrap_or_else(|| {
loss_compensated_repair_target(base_deficit_symbols, need.round_loss_fraction)
})
}
fn repair_block_symbol_count(requests: &[QuicBlockRepairRequest]) -> u64 {
requests.iter().fold(0u64, |acc, request| {
acc.saturating_add(u64::from(request.symbols))
})
}
fn next_feedback_round_or_no_convergence(
feedback_rounds: u32,
max_feedback_rounds: u32,
pending_entries: usize,
) -> Result<u32, QuicTransportError> {
let fail_closed_rounds = still_viable_feedback_fail_closed_rounds(max_feedback_rounds);
if pending_entries > 0 && feedback_rounds >= fail_closed_rounds {
return Err(QuicTransportError::NoConvergence {
rounds: feedback_rounds,
pending: pending_entries,
});
}
Ok(feedback_rounds.saturating_add(1))
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
struct SentControlStreamFrame {
stream: StreamId,
offset: u64,
/// Stream payload bytes carried by this frame, so ACK processing can
/// clock the delivery-rate estimator for adaptive source-stream pacing.
len: u64,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
struct SentDatagramPacket {
payload_bytes: u64,
time_sent_micros: u64,
}
#[derive(Debug, Clone)]
struct ReceivedSourceStreamFrame {
offset: u64,
data: Bytes,
fin: bool,
}
fn acked_packet_ranges_from_frames(
frames: &[QuicFrame],
) -> Result<Vec<NativeAckRange>, NativeQuicConnectionError> {
let mut ranges = Vec::new();
for frame in frames {
if let QuicFrame::Ack {
largest_acknowledged,
first_ack_range,
ack_ranges,
..
} = frame
{
ranges.extend(ack_frame_packet_ranges(
largest_acknowledged.value(),
first_ack_range.value(),
ack_ranges,
)?);
}
}
Ok(ranges)
}
fn ack_frame_packet_ranges(
largest_acknowledged: u64,
first_ack_range: u64,
ack_ranges: &[crate::net::atp::protocol::quic_frames::AckRange],
) -> Result<Vec<NativeAckRange>, NativeQuicConnectionError> {
let first_smallest = largest_acknowledged.checked_sub(first_ack_range).ok_or(
NativeQuicConnectionError::InvalidState("ACK first range exceeds largest packet number"),
)?;
let mut ranges = vec![
NativeAckRange::new(largest_acknowledged, first_smallest).ok_or(
NativeQuicConnectionError::InvalidState("invalid ACK first range"),
)?,
];
let mut previous_smallest = first_smallest;
for range in ack_ranges {
let next_largest = previous_smallest
.checked_sub(range.gap.value().saturating_add(2))
.ok_or(NativeQuicConnectionError::InvalidState(
"ACK range gap underflowed packet number space",
))?;
let next_smallest = next_largest
.checked_sub(range.ack_range_length.value())
.ok_or(NativeQuicConnectionError::InvalidState(
"ACK range length exceeds largest packet number",
))?;
ranges.push(
NativeAckRange::new(next_largest, next_smallest)
.ok_or(NativeQuicConnectionError::InvalidState("invalid ACK range"))?,
);
previous_smallest = next_smallest;
}
Ok(ranges)
}
fn packet_in_ack_ranges(packet_number: u64, ranges: &[NativeAckRange]) -> bool {
ranges
.iter()
.any(|range| packet_number >= range.smallest && packet_number <= range.largest)
}
const QUIC_SOURCE_STREAM_FAST_RETRANSMIT_PACKET_THRESHOLD: u64 = 3;
/// Recovery drains feed `requeue_sent_stream_frame` + a paced `flush`, so these
/// caps bound how much recovery data becomes AVAILABLE to the pacer per event —
/// they are not a wire burst. Asymmetric by design (MATRIX-210 A/B): the
/// ACK-gap FAST drain uses a packet-threshold detector that fires spuriously
/// under ACK batching, and a 32-packet drain amplified each false positive
/// into seconds of retransmit churn (50M/good 3.45→26s, 500M/perfect
/// 9.5→21.9s reps) — so it stays small. The PTO drain is timer-based (no
/// spurious trigger) and 64 packets/200ms capped loss recovery at ~2.5MB/s;
/// 256 lifts the recovery ceiling where it is safe (oh6gm2 phase 2).
const QUIC_SOURCE_STREAM_FAST_RETRANSMIT_MAX_PACKETS: usize = 8;
/// Drain cap when the gap-detected set itself is large (> the base cap):
/// spurious ACK-batching detections are 1-3 packet singles, so a big
/// detected set is real burst loss and gets drained in few rounds instead
/// of 8 packets per PTO-clocked firing.
const QUIC_SOURCE_STREAM_FAST_RETRANSMIT_BURST_MAX_PACKETS: usize = 64;
const QUIC_SOURCE_STREAM_PTO_RETRANSMIT_MAX_PACKETS: usize = 256;
fn packet_lost_by_ack_gap(packet_number: u64, acked_ranges: &[NativeAckRange]) -> bool {
let Some(largest_acked) = acked_ranges.iter().map(|range| range.largest).max() else {
return false;
};
packet_number.saturating_add(QUIC_SOURCE_STREAM_FAST_RETRANSMIT_PACKET_THRESHOLD)
<= largest_acked
&& !packet_in_ack_ranges(packet_number, acked_ranges)
}
fn dedup_stream_frames_for_retransmit(
mut frames: Vec<SentControlStreamFrame>,
) -> Vec<SentControlStreamFrame> {
frames.sort_by_key(|frame| (frame.stream.0, frame.offset));
frames.dedup();
frames
}
struct NativeQuicSpraySymbol {
symbol: Symbol,
entry: u32,
auth_tag: Option<[u8; TAG_SIZE]>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum NeedMorePtoMode {
RetransmitRecorded,
SendFresh,
}
fn need_more_pto_mode(need_frames: &[SentControlStreamFrame]) -> NeedMorePtoMode {
if need_frames.is_empty() {
NeedMorePtoMode::SendFresh
} else {
NeedMorePtoMode::RetransmitRecorded
}
}
fn feedback_round_for_need_or_no_convergence(
feedback_rounds: u32,
max_feedback_rounds: u32,
requested_feedback_round: u32,
pending_entries: usize,
) -> Result<(u32, u32), QuicTransportError> {
let next_feedback_round = feedback_rounds.saturating_add(1);
let response_feedback_round = if requested_feedback_round == 0 {
next_feedback_round
} else {
requested_feedback_round
};
let fail_closed_rounds = still_viable_feedback_fail_closed_rounds(max_feedback_rounds);
if pending_entries > 0 && response_feedback_round > fail_closed_rounds {
return Err(QuicTransportError::NoConvergence {
rounds: feedback_rounds,
pending: pending_entries,
});
}
Ok((
feedback_rounds.max(response_feedback_round),
response_feedback_round,
))
}
fn still_viable_feedback_fail_closed_rounds(max_feedback_rounds: u32) -> u32 {
max_feedback_rounds.saturating_add(QUIC_STILL_VIABLE_FEEDBACK_GRACE_ROUNDS)
}
fn send_native_object_complete_for_round(
cx: &Cx,
conn: &mut NativeQuicConnection,
control: &mut NativeQuicFrameTransport,
round: u32,
round_symbols_sent: u64,
) -> Result<(), QuicTransportError> {
control.send_json(
cx,
conn,
FrameType::ObjectComplete,
&super::QuicRoundComplete {
round,
round_symbols_sent,
},
)
}
fn trace_stale_round_complete(cx: &Cx, expected_round: u32, got: &super::QuicRoundComplete) {
let expected_round_s = expected_round.to_string();
let got_round_s = got.round.to_string();
let got_symbols_s = got.round_symbols_sent.to_string();
cx.trace_with_fields(
"atp_quic.receive.stale_round_complete",
&[
("expected_round", expected_round_s.as_str()),
("got_round", got_round_s.as_str()),
("round_symbols_sent", got_symbols_s.as_str()),
],
);
quic_rqtrace!(
"receiver: stale ObjectComplete expected_round={} got_round={} round_symbols_sent={}",
expected_round,
got.round,
got.round_symbols_sent,
);
}
fn trace_native_repair_accounting(
cx: &Cx,
direction: &'static str,
feedback_round: u32,
base_deficit_symbols: u64,
requested_repair_symbols: u64,
loss_compensated_target_symbols: u64,
emitted_repair_symbols: Option<u64>,
need: &QuicNeedMore,
) {
let feedback_round_s = feedback_round.to_string();
let base_deficit_symbols_s = base_deficit_symbols.to_string();
let requested_repair_symbols_s = requested_repair_symbols.to_string();
let loss_compensated_target_symbols_s = loss_compensated_target_symbols.to_string();
let emitted_repair_symbols_s = emitted_repair_symbols
.map(|value| value.to_string())
.unwrap_or_else(|| "pending".to_string());
let gap_to_target_symbols = emitted_repair_symbols
.map(|emitted| loss_compensated_target_symbols.saturating_sub(emitted))
.unwrap_or_else(|| loss_compensated_target_symbols.saturating_sub(requested_repair_symbols))
.to_string();
let round_loss_fraction_s = format!("{:.6}", need.round_loss_fraction.unwrap_or(0.0));
let repair_blocks_s = need.repair_blocks.len().to_string();
let source_requests_s = need.source_symbols.len().to_string();
cx.trace_with_fields(
"atp_quic.repair_accounting",
&[
("transport", "quic"),
("direction", direction),
("feedback_round", feedback_round_s.as_str()),
("base_deficit_symbols", base_deficit_symbols_s.as_str()),
(
"requested_repair_symbols",
requested_repair_symbols_s.as_str(),
),
(
"loss_compensated_target_symbols",
loss_compensated_target_symbols_s.as_str(),
),
("emitted_repair_symbols", emitted_repair_symbols_s.as_str()),
("gap_to_target_symbols", gap_to_target_symbols.as_str()),
("round_loss_fraction", round_loss_fraction_s.as_str()),
("repair_blocks", repair_blocks_s.as_str()),
("source_requests", source_requests_s.as_str()),
],
);
}
// `Sync` on the proof payload keeps the returned future `Send`: `&T` is only `Send`
// when `T: Sync`, and every caller already passes a plain serializable record.
async fn send_native_proof_until_close<T: serde::Serialize + Sync>(
cx: &Cx,
link: &mut QuicLink,
control: &mut NativeQuicFrameTransport,
proof: &T,
receipt: &ReceiveReceipt,
config: &QuicConfig,
) -> Result<(), QuicTransportError> {
let proof_frame = super::json_frame(FrameType::Proof, proof)?;
control.send(cx, &mut link.conn, &proof_frame)?;
let proof_packets = link.flush(cx).await?;
let proof_frames = link.last_flushed_stream_frames();
// Happy path of every transfer: trace-gated, since `wait_diagnostics`
// reads /proc and this would otherwise print to stderr on each run.
quic_rqtrace!(
"receiver: proof_sent packets={proof_packets} stream_frames={} {}",
proof_frames.len(),
link.wait_diagnostics()
);
let max_attempts =
TERMINAL_PROOF_RETRANSMIT_ATTEMPTS.min(needmore_pto_attempt_budget(config.idle_timeout));
let mut attempts = 0u32;
loop {
cx.checkpoint().map_err(|_| QuicTransportError::Cancelled)?;
if let Some(frame) = control.try_recv(cx, &mut link.conn)? {
match frame.frame_type() {
FrameType::Close => return Ok(()),
FrameType::KeepAlive | FrameType::ObjectComplete => continue,
FrameType::ObjectRequest | FrameType::ObjectManifest => {
link.retransmit_stream_frames(cx, &proof_frames, "terminal_proof_request")
.await?;
continue;
}
got => {
return Err(QuicTransportError::Unexpected {
got,
expected: "Close | KeepAlive | ObjectComplete | ObjectRequest | ObjectManifest",
});
}
}
}
link.flush(cx).await?;
if link.pump_inbound_for(cx, NEEDMORE_PTO).await? > 0 {
continue;
}
if attempts >= max_attempts {
super::quic_progress(format_args!(
"receiver: proof_wait_exhausted attempts={attempts} {}",
link.wait_diagnostics()
));
return Ok(());
}
attempts = attempts.saturating_add(1);
quic_rqtrace!(
"receiver: Proof PTO retransmit attempt={} max_attempts={} stream_frames={} committed={} feedback_rounds={} symbols_accepted={}",
attempts,
max_attempts,
proof_frames.len(),
receipt.committed,
receipt.feedback_rounds,
receipt.symbols_accepted,
);
link.retransmit_stream_frames(cx, &proof_frames, "terminal_proof_pto")
.await?;
}
}
fn same_need_more_request_shape(left: &QuicNeedMore, right: &QuicNeedMore) -> bool {
left.feedback_round == right.feedback_round
&& left.pending == right.pending
&& left.repair_blocks == right.repair_blocks
&& left.source_symbols == right.source_symbols
}
fn drop_duplicate_need_more_frames(
pending: &mut VecDeque<Frame>,
served_need: &QuicNeedMore,
) -> Result<usize, QuicTransportError> {
let mut retained = VecDeque::with_capacity(pending.len());
let mut dropped = 0usize;
while let Some(frame) = pending.pop_front() {
if frame.frame_type() == FrameType::ObjectRequest {
let queued = super::parse_json::<QuicNeedMore>(&frame)?;
if same_need_more_request_shape(&queued, served_need) {
dropped = dropped.saturating_add(1);
continue;
}
}
retained.push_back(frame);
}
*pending = retained;
Ok(dropped)
}
fn repair_block_trace_summary(requests: &[QuicBlockRepairRequest]) -> String {
const MAX_DETAIL_BLOCKS: usize = 16;
let mut max_symbols = 0u32;
let mut parts = Vec::new();
for request in requests.iter().take(MAX_DETAIL_BLOCKS) {
max_symbols = max_symbols.max(request.symbols);
parts.push(format!(
"{}:{}:{}",
request.entry, request.sbn, request.symbols
));
}
for request in requests.iter().skip(MAX_DETAIL_BLOCKS) {
max_symbols = max_symbols.max(request.symbols);
}
if requests.len() > MAX_DETAIL_BLOCKS {
parts.push(format!("+{}more", requests.len() - MAX_DETAIL_BLOCKS));
}
if parts.is_empty() {
"none".to_string()
} else {
format!("max={} [{}]", max_symbols, parts.join(","))
}
}
fn trace_repair_block_deficits(direction: &str, round: u32, requests: &[QuicBlockRepairRequest]) {
if std::env::var_os("ATP_RQ_TRACE").is_none() {
return;
}
for request in requests {
eprintln!(
"[ATP_RQ_TRACE] [atp-quic] {direction}: NeedMoreBlock round={round} entry={} sbn={} requested_symbols={}",
request.entry, request.sbn, request.symbols
);
}
}
/// Monotonic data-plane clock step (microseconds) fed to the connection per pump
/// operation. The transfer's correctness does not depend on real time; this only
/// keeps the connection's loss/ACK bookkeeping monotonic.
const CLOCK_STEP_MICROS: u64 = 1_000;
/// Client-side TLS material for a native QUIC connection.
///
/// Built by the caller via [`crate::net::quic_native::handshake_driver::client_config`]
/// (or any TLS-1.3 `rustls::ClientConfig` advertising the ATP-over-QUIC ALPN) and
/// the server name to verify. There is no insecure skip-verify path: the
/// configured roots gate the handshake.
#[derive(Clone)]
pub struct QuicClientTls {
/// Server name verified against the presented certificate (WebPKI).
pub server_name: ServerName<'static>,
/// TLS-1.3 client configuration (root trust + ALPN).
pub config: Arc<ClientConfig>,
}
impl std::fmt::Debug for QuicClientTls {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("QuicClientTls")
.field("server_name", &self.server_name)
.field("config", &"<rustls::ClientConfig>")
.finish()
}
}
/// Server-side TLS material for a native QUIC connection.
///
/// Built by the caller via [`crate::net::quic_native::handshake_driver::server_config`]
/// (or any TLS-1.3 `rustls::ServerConfig` presenting a certificate chain and key
/// and advertising the ATP-over-QUIC ALPN).
#[derive(Clone)]
pub struct QuicServerTls {
/// TLS-1.3 server configuration (certificate chain + private key + ALPN).
pub config: Arc<ServerConfig>,
}
impl std::fmt::Debug for QuicServerTls {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("QuicServerTls")
.field("config", &"<rustls::ServerConfig>")
.finish()
}
}
/// Convert an [`AtpPacketProtection`] outcome into a transport result.
fn protection_result<T>(
outcome: Outcome<T, crate::net::atp::protocol::outcome::AtpError>,
) -> Result<T, QuicTransportError> {
match outcome {
Outcome::Ok(value) => Ok(value),
Outcome::Err(err) => Err(QuicTransportError::Quic(format!(
"packet protection: {err:?}"
))),
Outcome::Cancelled(_) => Err(QuicTransportError::Cancelled),
Outcome::Panicked(_) => Err(QuicTransportError::Quic(
"packet protection panicked".to_string(),
)),
}
}
fn map_udp_error(err: crate::net::quic_native::QuicUdpEndpointError) -> QuicTransportError {
match err {
crate::net::quic_native::QuicUdpEndpointError::Cancelled => QuicTransportError::Cancelled,
other => QuicTransportError::Quic(format!("udp endpoint: {other}")),
}
}
fn map_tls_error(err: crate::net::quic_native::QuicTlsError) -> QuicTransportError {
QuicTransportError::Quic(format!("quic handshake: {err}"))
}
/// Encode the simplified 1-RTT data-plane header for `packet_number`.
fn encode_one_rtt_header(packet_number: u64) -> [u8; ONE_RTT_HEADER_LEN] {
let mut header = [0u8; ONE_RTT_HEADER_LEN];
header[0] = ONE_RTT_FIXED_BIT; // key_phase 0
header[1..].copy_from_slice(&packet_number.to_be_bytes());
header
}
/// Decode a received 1-RTT data-plane packet into `(key_phase, packet_number,
/// header_bytes, ciphertext, tag)`. Returns `None` for any packet that is not a
/// well-formed short-header 1-RTT packet (e.g. a late handshake long-header
/// retransmit), which the caller silently drops, matching QUIC semantics.
///
/// The live receive path uses [`parse_one_rtt_header`] + in-place unprotection
/// instead; this borrowing decomposition remains for header-shape unit tests.
#[cfg(test)]
fn decode_one_rtt_packet(
packet: &[u8],
) -> Option<(bool, u64, &[u8], &[u8], [u8; ONE_RTT_TAG_LEN])> {
if packet.len() < ONE_RTT_HEADER_LEN + ONE_RTT_TAG_LEN {
return None;
}
let flags = packet[0];
// A 1-RTT short header has the QUIC fixed bit (0x40) set and the long-header
// form bit (0x80) clear. Reject long-header packets (e.g. a late handshake
// retransmit arriving on this socket) up front, consistent with
// `is_long_header`, rather than relying on AEAD failure to drop them.
if flags & 0x80 != 0 || flags & ONE_RTT_FIXED_BIT == 0 {
return None;
}
let key_phase = flags & ONE_RTT_KEY_PHASE_BIT != 0;
let mut pn_bytes = [0u8; 8];
pn_bytes.copy_from_slice(&packet[1..ONE_RTT_HEADER_LEN]);
let packet_number = u64::from_be_bytes(pn_bytes);
let header = &packet[..ONE_RTT_HEADER_LEN];
let body = &packet[ONE_RTT_HEADER_LEN..];
let tag_offset = body.len() - ONE_RTT_TAG_LEN;
let mut tag = [0u8; ONE_RTT_TAG_LEN];
tag.copy_from_slice(&body[tag_offset..]);
let ciphertext = &body[..tag_offset];
Some((key_phase, packet_number, header, ciphertext, tag))
}
/// A received 1-RTT packet after authentication and a single frame decode.
///
/// `frames` hold zero-copy `Bytes` views into the (decrypted-in-place)
/// datagram buffer, so stashing a decoded packet under receive backpressure
/// keeps the authenticated result instead of re-running AEAD on the raw
/// datagram — which the anti-replay window would (correctly) reject.
struct DecodedOneRttPacket {
packet_number: u64,
frames: Vec<QuicFrame>,
}
/// Outcome of decoding + authenticating one received datagram.
enum InboundPacketDecode {
/// Authenticated 1-RTT packet, frames decoded once.
Decoded(DecodedOneRttPacket),
/// Not a well-formed 1-RTT short-header packet (silently dropped).
NotOneRtt,
/// Authentication/replay failure (silently dropped, per QUIC).
Dropped,
}
enum InboundPacketDisposition {
Applied,
DatagramBackpressure,
ReassemblyDropped,
}
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
struct IngestPacketsReport {
packets_consumed: usize,
one_rtt_packets_processed: usize,
receive_backpressure: bool,
}
/// Parse the simplified 1-RTT short header, returning `(key_phase,
/// packet_number)` without borrowing slices out of the packet.
fn parse_one_rtt_header(packet: &[u8]) -> Option<(bool, u64)> {
if packet.len() < ONE_RTT_HEADER_LEN + ONE_RTT_TAG_LEN {
return None;
}
let flags = packet[0];
if flags & 0x80 != 0 || flags & ONE_RTT_FIXED_BIT == 0 {
return None;
}
let key_phase = flags & ONE_RTT_KEY_PHASE_BIT != 0;
let mut pn_bytes = [0u8; 8];
pn_bytes.copy_from_slice(&packet[1..ONE_RTT_HEADER_LEN]);
Some((key_phase, u64::from_be_bytes(pn_bytes)))
}
fn one_rtt_max_payload_for_udp_packet(max_udp_packet: usize) -> usize {
max_udp_packet
.saturating_sub(ONE_RTT_PACKET_OVERHEAD)
.saturating_sub(ONE_RTT_COALESCED_CONTROL_HEADROOM)
}
fn coalesced_datagram_frames_per_packet(
max_app_payload: usize,
datagram_frame_len: usize,
) -> usize {
(max_app_payload / datagram_frame_len.max(1)).max(1)
}
fn frame_is_ack_eliciting_for_recovery(frame: &QuicFrame) -> bool {
!matches!(
frame,
QuicFrame::Padding { .. } | QuicFrame::Ack { .. } | QuicFrame::ConnectionClose { .. }
)
}
fn frames_have_datagram(frames: &[QuicFrame]) -> bool {
frames
.iter()
.any(|frame| matches!(frame, QuicFrame::Datagram { .. }))
}
fn datagram_frame_count(frames: &[QuicFrame]) -> usize {
frames
.iter()
.filter(|frame| matches!(frame, QuicFrame::Datagram { .. }))
.count()
}
fn frames_require_quic_recovery_in_flight(frames: &[QuicFrame]) -> bool {
frames.iter().any(|frame| {
frame_is_ack_eliciting_for_recovery(frame) && !matches!(frame, QuicFrame::Datagram { .. })
})
}
fn data_plane_packet_accounting_bytes(packet_len: usize) -> u64 {
u64::try_from(packet_len)
.unwrap_or(u64::MAX)
.clamp(1, QUIC_DATA_PLANE_TELEMETRY_PACKET_BYTES)
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
struct DataPlaneCwndTelemetry {
bytes_in_flight: u64,
congestion_window: u64,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
struct DataPlaneFlushAdmission {
max_frame_bytes: usize,
cwnd_telemetry: Option<DataPlaneCwndTelemetry>,
}
fn data_plane_cwnd_telemetry(
transport: &crate::net::quic_native::QuicTransportMachine,
pending_datagrams: usize,
) -> Option<DataPlaneCwndTelemetry> {
if pending_datagrams == 0 || transport.can_send(QUIC_DATA_PLANE_TELEMETRY_PACKET_BYTES) {
return None;
}
Some(DataPlaneCwndTelemetry {
bytes_in_flight: transport.bytes_in_flight(),
congestion_window: transport.congestion_window_bytes(),
})
}
fn data_plane_flush_admission(
transport: &crate::net::quic_native::QuicTransportMachine,
pending_datagrams: usize,
spray_frame_payload_limit: usize,
max_app_payload: usize,
) -> DataPlaneFlushAdmission {
DataPlaneFlushAdmission {
max_frame_bytes: if pending_datagrams > 0 {
spray_frame_payload_limit
} else {
max_app_payload
},
cwnd_telemetry: data_plane_cwnd_telemetry(transport, pending_datagrams),
}
}
fn data_plane_packet_uses_paced_recovery(frames: &[QuicFrame]) -> bool {
frames_have_datagram(frames) && !frames_require_quic_recovery_in_flight(frames)
}
fn source_stream_packet_uses_paced_recovery(
frames: &[QuicFrame],
source_stream: Option<StreamId>,
) -> bool {
// The reliable source STREAM is DELIBERATELY pacer-governed with in_flight=false, NOT
// NewReno-cwnd-clocked. MATRIX-202 measured why: cwnd/ACK-clocking it beats the pacer on a
// perfectly-clean link (500M 14s→8.9s) but CATASTROPHICALLY regresses mildly-lossy links
// (50M/good @0.1% loss/25ms: 4s→~52s, ~12×) because NewReno halves cwnd on every drop and
// recovers a full RTT at a time, while fixed-rate pacing + ack-gap retransmit shrugs off
// mild loss. The near-clean source-stream selection still admits 0.1% loss, so the pacer is
// the correct admission control. (Do not re-clock this on cwnd without also solving the
// mild-loss cwnd-collapse — that is the QUIC congestion owner's call, br-asupersync-uw1cc2.)
let Some(source_stream) = source_stream else {
return false;
};
let mut has_source_stream_data = false;
for frame in frames {
match frame {
QuicFrame::Stream { stream_id, .. } if stream_id.value() == source_stream.0 => {
has_source_stream_data = true;
}
frame if frame_is_ack_eliciting_for_recovery(frame) => return false,
_ => {}
}
}
has_source_stream_data
}
fn packet_uses_paced_recovery(frames: &[QuicFrame], source_stream: Option<StreamId>) -> bool {
data_plane_packet_uses_paced_recovery(frames)
|| source_stream_packet_uses_paced_recovery(frames, source_stream)
}
fn packet_tracks_recovery_in_flight(frames: &[QuicFrame], source_stream: Option<StreamId>) -> bool {
!packet_uses_paced_recovery(frames, source_stream)
&& frames.iter().any(frame_is_ack_eliciting_for_recovery)
}
fn trace_quic_data_plane_cwnd_telemetry(
cx: &Cx,
pending_datagrams: usize,
telemetry: DataPlaneCwndTelemetry,
) {
if std::env::var_os("ATP_RQ_TRACE").is_none() {
return;
}
let pending_datagrams_s = pending_datagrams.to_string();
let bytes_in_flight_s = telemetry.bytes_in_flight.to_string();
let congestion_window_s = telemetry.congestion_window.to_string();
cx.trace_with_fields(
"atp_quic.sender.data_plane_cwnd_telemetry",
&[
("pending_datagrams", pending_datagrams_s.as_str()),
("bytes_in_flight", bytes_in_flight_s.as_str()),
("congestion_window", congestion_window_s.as_str()),
("admission", "pacer"),
],
);
quic_rqtrace!(
"sender: data_plane_cwnd_telemetry pending_datagrams={} bytes_in_flight={} congestion_window={} admission=pacer",
pending_datagrams,
telemetry.bytes_in_flight,
telemetry.congestion_window,
);
}
fn trace_quic_data_plane_loss_timeout(
cx: &Cx,
pending_datagrams: usize,
lost_packets: usize,
lost_bytes: u64,
bytes_in_flight: u64,
congestion_window: u64,
pto_count: u32,
) {
let pending_datagrams_s = pending_datagrams.to_string();
let lost_packets_s = lost_packets.to_string();
let lost_bytes_s = lost_bytes.to_string();
let bytes_in_flight_s = bytes_in_flight.to_string();
let congestion_window_s = congestion_window.to_string();
let pto_count_s = pto_count.to_string();
cx.trace_with_fields(
"atp_quic.sender.data_plane_loss_timeout",
&[
("pending_datagrams", pending_datagrams_s.as_str()),
("lost_packets", lost_packets_s.as_str()),
("lost_bytes", lost_bytes_s.as_str()),
("bytes_in_flight", bytes_in_flight_s.as_str()),
("congestion_window", congestion_window_s.as_str()),
("pto_count", pto_count_s.as_str()),
],
);
quic_rqtrace!(
"sender: data_plane_loss_timeout pending_datagrams={} lost_packets={} lost_bytes={} bytes_in_flight={} congestion_window={} pto_count={}",
pending_datagrams,
lost_packets,
lost_bytes,
bytes_in_flight,
congestion_window,
pto_count,
);
}
fn trace_quic_data_plane_pacer_limited(
cx: &Cx,
pending_datagrams: usize,
wait: Duration,
pacing_rate_bps: u64,
bytes_in_flight: u64,
congestion_window: u64,
) {
if std::env::var_os("ATP_RQ_TRACE").is_none() {
return;
}
let pending_datagrams_s = pending_datagrams.to_string();
let wait_micros_s = wait.as_micros().to_string();
let pacing_rate_bps_s = pacing_rate_bps.to_string();
let bytes_in_flight_s = bytes_in_flight.to_string();
let congestion_window_s = congestion_window.to_string();
cx.trace_with_fields(
"atp_quic.sender.data_plane_pacer_limited",
&[
("pending_datagrams", pending_datagrams_s.as_str()),
("wait_micros", wait_micros_s.as_str()),
("pacing_rate_bps", pacing_rate_bps_s.as_str()),
("bytes_in_flight", bytes_in_flight_s.as_str()),
("congestion_window", congestion_window_s.as_str()),
("admission", "pacer"),
],
);
quic_rqtrace!(
"sender: data_plane_pacer_limited pending_datagrams={} wait_micros={} pacing_rate_bps={} bytes_in_flight={} congestion_window={} admission=pacer",
pending_datagrams,
wait.as_micros(),
pacing_rate_bps,
bytes_in_flight,
congestion_window,
);
}
fn promote_source_stream_pacing(
pacing: &mut QuicSprayPacingDecision,
config: &QuicConfig,
symbol_datagram_frame_len: usize,
) {
if config.bwlimit_bps.is_none() && super::quic_near_clean_source_stream_enabled(config, pacing)
{
let max_rate = super::QUIC_RELIABLE_SOURCE_STREAM_MAX_PACING_BPS;
if pacing.pacing_rate_bps < max_rate {
pacing.pacing_rate_bps = max_rate;
super::update_quic_pacing_pause(
pacing,
symbol_datagram_frame_len,
config.max_spray_symbols_per_flush,
);
}
}
}
fn enforce_native_repair_round_pacing(
pacing: &mut QuicSprayPacingDecision,
symbol_datagram_frame_len: usize,
) {
let frame_len = symbol_datagram_frame_len.max(1) as f64;
let repair_rtt_s = pacing
.path_rtt_s
.max(Duration::from_micros(QUIC_LOSSY_COLD_START_RTT_MICROS).as_secs_f64())
.max(0.001);
let bdp_symbols = ((pacing.pacing_rate_bps.max(1) as f64 * repair_rtt_s) / frame_len)
.ceil()
.max(1.0) as usize;
pacing.max_burst_symbols = 1;
pacing.burst_cap_share_symbols = 1;
pacing.cwnd_share_symbols = bdp_symbols.max(1);
pacing.cwnd_symbols = pacing.cwnd_symbols.max(pacing.cwnd_share_symbols);
let repair_cwnd_bytes = u64::try_from(pacing.cwnd_share_symbols)
.unwrap_or(u64::MAX)
.saturating_mul(u64::try_from(symbol_datagram_frame_len.max(1)).unwrap_or(u64::MAX));
pacing.path_cwnd_bytes = pacing.path_cwnd_bytes.max(repair_cwnd_bytes);
super::update_quic_pacing_pause(pacing, symbol_datagram_frame_len, 1);
}
fn queued_fountain_feedback_count(pending: &VecDeque<Frame>) -> usize {
pending
.iter()
.filter(|frame| {
matches!(
frame.frame_type(),
FrameType::ObjectRequest | FrameType::Proof
)
})
.count()
}
fn trace_quic_initial_spray_cut_for_feedback(
cx: &Cx,
sent_symbols: u64,
queued_feedback: usize,
pending_datagrams: usize,
bytes_in_flight: u64,
congestion_window: u64,
entry_index: u32,
sbn: u8,
repair_cursor: usize,
) {
let sent_symbols_s = sent_symbols.to_string();
let queued_feedback_s = queued_feedback.to_string();
let pending_datagrams_s = pending_datagrams.to_string();
let bytes_in_flight_s = bytes_in_flight.to_string();
let congestion_window_s = congestion_window.to_string();
let entry_index_s = entry_index.to_string();
let sbn_s = sbn.to_string();
let repair_cursor_s = repair_cursor.to_string();
cx.trace_with_fields(
"atp_quic.sender.initial_spray_feedback_cut",
&[
("sent_symbols", sent_symbols_s.as_str()),
("queued_feedback", queued_feedback_s.as_str()),
("pending_datagrams", pending_datagrams_s.as_str()),
("bytes_in_flight", bytes_in_flight_s.as_str()),
("congestion_window", congestion_window_s.as_str()),
("entry", entry_index_s.as_str()),
("sbn", sbn_s.as_str()),
("repair_cursor", repair_cursor_s.as_str()),
],
);
quic_rqtrace!(
"sender: initial_spray_feedback_cut sent_symbols={} queued_feedback={} pending_datagrams={} bytes_in_flight={} congestion_window={} entry={} sbn={} repair_cursor={}",
sent_symbols,
queued_feedback,
pending_datagrams,
bytes_in_flight,
congestion_window,
entry_index,
sbn,
repair_cursor,
);
}
/// Delivery-clocked pacing-rate follower for the reliable source stream.
///
/// BBR-shaped, deliberately tiny: a windowed **max** filter over recent
/// per-ACK-window delivery-rate samples estimates the bottleneck bandwidth,
/// and the pacing rate follows `1.25 × max_filter` — probing upward on clean
/// windows and holding ~link rate under sustained *random* loss (where
/// delivered ≈ 0.9 × sent), while genuine congestion (a sustained delivery
/// collapse) ages the old maximum out of the filter within
/// `STREAM_RATE_FILTER_WINDOWS` windows and pulls the rate down with it.
///
/// This replaces the fixed `QUIC_RELIABLE_SOURCE_STREAM_MAX_PACING_BPS`
/// setpoint (which capped clean encrypted throughput at ~37 MB/s) and avoids
/// both NewReno's mild-loss cwnd collapse (MATRIX-202) and min-ratchet
/// delivery followers, whose bottleneck estimate can only fall when no
/// loss-free window ever arrives (10% regimes).
struct SourceStreamRatePacer {
rate_bytes_per_s: u64,
/// Regime-derived starting rate; also the recovery floor while the link
/// shows no sustained loss, so a post-overrun window can never crawl
/// below the path's conservative estimate (measured delivery during PTO
/// recovery reflects only the small retransmit batches, not the path).
initial_rate_bytes_per_s: u64,
/// Cumulative bytes queued for retransmission. Once this passes
/// [`STREAM_RATE_FLOOR_RELEASE_LOST_BYTES`] the initial-rate floor is
/// released: a seed rate above the true link rate otherwise pins every
/// recovery burst at the mis-seeded rate forever (measured: an 80 MiB/s
/// seed on a 25 MB/s shaped link re-dropped its own retransmit tails for
/// ~87 consecutive PTO rounds), while the max-filter alone tracks the
/// real delivery rate within 8 windows.
total_lost_bytes: u64,
delivery_samples: [u64; STREAM_RATE_FILTER_WINDOWS],
next_sample: usize,
/// Index into [`STREAM_RATE_PROBE_CYCLE_GAINS_X1000`] — the PROBE_BW
/// gain cycle (MATRIX-227). The 0.75× drain phase is the EXPLICIT
/// replacement for what the consumption-clocked flow-window stall was
/// doing implicitly (the M224/225/226 drain-phase law): a constant
/// 1.25× probe over-offers continuously and, with nothing draining the
/// shaper queue, re-sends most of the payload.
cycle_phase: usize,
/// Wall micros when the current gain phase started (caller-supplied
/// clock, same domain as the delivery sampler).
phase_started_micros: u64,
}
/// Recent-delivery max-filter depth.
const STREAM_RATE_FILTER_WINDOWS: usize = 8;
/// Pacing floor; loss recovery must always trickle.
const STREAM_RATE_MIN_BYTES_PER_S: u64 = 256 * 1024;
/// Pacing ceiling (~4 gbit/s payload) — far above the bench links so the
/// delivery filter, not this constant, is what binds.
const STREAM_RATE_MAX_BYTES_PER_S: u64 = 512 * 1024 * 1024;
/// Probe/headroom gain over the max-filtered delivery estimate (BBR form:
/// `pacing = gain × BtlBw`). Growth toward the link is geometric with ratio
/// `gain × pacer_efficiency`, so the flush pipeline must stay efficient
/// (see the zero-timeout opportunistic pump in the flush loop) — an
/// unbounded multiplicative probe instead overruns the shaper by 4× and
/// collapses into retransmit churn (measured: 500M 11s → 53s; that
/// refutation predates the bounded receive window). REFUTED (oh6gm2 ph3,
/// 2026-07-06): raising the gain to 1.5 measured FLAT on 500M/perfect
/// (6.75 vs 6.66 med, 3 reps each, contemporaneous) — flush efficiency
/// drops in proportion, so the clean-large equilibrium sits at
/// `gain × efficiency ≈ 1` regardless of the gain. The binding constraint
/// is pipeline efficiency, not probe headroom; raise burst amortization,
/// not this constant.
const STREAM_RATE_GAIN_X1000: u64 = 1250;
/// PROBE_BW pacing-gain cycle (MATRIX-227, BBR §4.3.4.3): one probe phase
/// discovers headroom, one drain phase empties the queue the probe built,
/// six cruise phases sit at the estimate. Each phase lasts ~one RTprop.
/// Average gain 1.0 keeps the shaper queue empty at steady state — the
/// drain phase is the load-bearing element (see the drain-phase law notes
/// on `advance_bounded_recv_windows` and `SourceStreamRatePacer`).
const STREAM_RATE_PROBE_CYCLE_GAINS_X1000: [u64; 8] = [
STREAM_RATE_GAIN_X1000,
750,
1000,
1000,
1000,
1000,
1000,
1000,
];
/// Minimum gain-phase length when RTprop is unknown or implausibly small.
const STREAM_RATE_PHASE_MIN_MICROS: u64 = 25_000;
// NOTE (oh6gm2 ph3c refutation, 2026-07-07): a BBR-STARTUP-style schedule —
// gain 1.5 while the bottleneck estimate grows past the seed, permanent
// settle to 1.25 after 3 flat windows — hit its 50M/perfect target exactly
// (med 1.65→1.45, −12%) but REGRESSED 500M/perfect +17% (6.66→7.82 med;
// the hot climb overshoots the shaper at ~119 MB/s absolute rates and the
// recovery episodes cost more than the faster ramp saves). Reverted per the
// pre-registered guard rule. A future retry needs overshoot-aware exit
// (e.g. leave startup on FIRST loss evidence, not only on flat growth).
// RETRIED AND REFUTED AGAIN (oh6gm2 ph4, 2026-07-08): the overshoot-aware
// variant — gain 1.5, permanent exit on FIRST loss evidence OR the first
// armed window with delivery < 0.9 × offered — DID protect the equilibrium
// (500M/perfect 7.017 vs parent 7.055 med, contemporaneous same-night
// parent A/B) but regressed its own TARGET cell: 50M/perfect med 1.652 vs
// parent 1.452 (+14%), with the two worst walls (1.91 s, 2.05 s) appearing
// only in the climb build — the single hot window still buys an
// overshoot-loss episode whose recovery costs more than the faster ramp
// saves on a ~1.5 s transfer. Both startup shapes (flat-growth exit,
// overshoot-aware exit) are now refuted with mechanism: on this path the
// climb-phase gain lever LOSES; 50M/perfect variance is mode-lottery
// (0.95-1.65 s spread on the UNCHANGED parent), not ramp speed. Do not
// re-try startup gain schedules without a fundamentally different overshoot
// bound (e.g. absolute in-flight cap at wall contact, not gain shaping).
/// its rate at the FULL regime-derived seed: sustained loss is evidence the
/// seed overshoots the true link rate.
const STREAM_RATE_FLOOR_RELEASE_LOST_BYTES: u64 = 1024 * 1024;
/// After release the floor drops to `initial / 8`, NOT to the global
/// minimum: recovery-phase delivery samples reflect only the retransmit
/// trickle, so a released-to-minimum floor let the max-filter collapse the
/// rate to a crawl once the seeded samples rotated out (measured: a 256 KiB
/// release-to-minimum turned 6 MB tree streams on good into uniform ~22 s
/// walls and regressed 500M/good 46→53 s). An eighth of the seed still
/// clears a 4-8× mis-seed while keeping recovery paced at path scale.
const STREAM_RATE_FLOOR_RELEASE_DIVISOR: u64 = 8;
impl SourceStreamRatePacer {
fn new(initial_rate_bytes_per_s: u64) -> Self {
let rate = initial_rate_bytes_per_s
.clamp(STREAM_RATE_MIN_BYTES_PER_S, STREAM_RATE_MAX_BYTES_PER_S);
Self {
// Seed the filter with the initial rate so early jittery windows
// cannot pull the estimate below the regime-derived starting
// point; real samples overwrite these within 8 windows.
rate_bytes_per_s: rate,
initial_rate_bytes_per_s: rate,
total_lost_bytes: 0,
delivery_samples: [rate; STREAM_RATE_FILTER_WINDOWS],
next_sample: 0,
cycle_phase: 0,
phase_started_micros: 0,
}
}
/// Fold one delivery window (per-packet sampler output, MATRIX-226) into
/// the filter and return the new pacing rate:
/// `clamp(max(gain × max_filter(delivery), floor))`.
///
/// The floor is the regime-derived initial rate until cumulative
/// retransmit-queued bytes pass
/// [`STREAM_RATE_FLOOR_RELEASE_LOST_BYTES`]; sustained loss proves the
/// seed overshoots the true link rate, and keeping the floor there pins
/// every recovery burst above the link forever. Beyond that evidence the
/// max-filtered delivery estimate alone drives the rate (per-window
/// loss-reactive variants — settle gains, filter resets — were all
/// measured to interact badly with retransmit framing on shaped links,
/// so loss only ever releases the floor, never modulates the gain).
fn on_delivery_window(
&mut self,
window: DeliveryWindow,
lost_bytes: u64,
now_micros: u64,
rtprop_micros: Option<u64>,
) -> u64 {
// Samples come from the per-packet delivered-counter sampler
// (MATRIX-226) — honest by construction. The refuted alternatives
// are documented on SourceStreamDeliverySampler: ACK-window
// aggregates spike 3-4× on queue-drain clumps (MATRIX-224), and
// clamping those at the offered rate manufactures capacity evidence
// (MATRIX-225). App-limited samples may only RAISE the estimate
// (BBR rule): a chronically limited sender must not define its
// capacity downward. Windows with no completed clean flight leave
// the filter untouched — a stall no longer poisons the ring with
// near-zero aggregates (the pre-M226 source of recovery rate
// collapse the floor machinery papers over).
let current_bottleneck = self.delivery_samples.iter().copied().max().unwrap_or(0);
let raising_app_limited = window
.max_app_limited_bps
.filter(|sample| *sample > current_bottleneck);
let sample = match (window.max_bps, raising_app_limited) {
(Some(clean), Some(raise)) => Some(clean.max(raise)),
(Some(clean), None) => Some(clean),
(None, Some(raise)) => Some(raise),
(None, None) => None,
};
if let Some(sample) = sample {
self.delivery_samples[self.next_sample] = sample;
self.next_sample = (self.next_sample + 1) % STREAM_RATE_FILTER_WINDOWS;
}
let bottleneck = self.delivery_samples.iter().copied().max().unwrap_or(0);
self.total_lost_bytes = self.total_lost_bytes.saturating_add(lost_bytes);
// PROBE_BW gain cycle (MATRIX-227): advance one phase per RTprop.
// Phases only tick at folds — a stalled link freezes the cycle,
// which is correct (no traffic to shape).
let phase_len = rtprop_micros
.unwrap_or(STREAM_RATE_PHASE_MIN_MICROS)
.max(STREAM_RATE_PHASE_MIN_MICROS);
if now_micros.saturating_sub(self.phase_started_micros) >= phase_len {
self.cycle_phase = (self.cycle_phase + 1) % STREAM_RATE_PROBE_CYCLE_GAINS_X1000.len();
self.phase_started_micros = now_micros;
}
let gain_x1000 = STREAM_RATE_PROBE_CYCLE_GAINS_X1000[self.cycle_phase];
let floor = if self.total_lost_bytes > STREAM_RATE_FLOOR_RELEASE_LOST_BYTES {
(self.initial_rate_bytes_per_s / STREAM_RATE_FLOOR_RELEASE_DIVISOR)
.max(STREAM_RATE_MIN_BYTES_PER_S)
} else {
self.initial_rate_bytes_per_s
};
self.rate_bytes_per_s = bottleneck
.saturating_mul(gain_x1000)
.checked_div(1000)
.unwrap_or(bottleneck)
.max(floor)
.clamp(STREAM_RATE_MIN_BYTES_PER_S, STREAM_RATE_MAX_BYTES_PER_S);
self.rate_bytes_per_s
}
/// Current pacing-gain phase (index into the PROBE_BW cycle) — trace
/// observability for the wire engagement check.
fn cycle_phase(&self) -> usize {
self.cycle_phase
}
/// Current bottleneck-bandwidth estimate (the raw max-filtered delivery
/// rate, before the probe gain): the BtlBw term of the BDP in-flight cap.
fn bottleneck_bytes_per_s(&self) -> u64 {
self.delivery_samples.iter().copied().max().unwrap_or(0)
}
}
/// Per-packet delivery-rate sampling for the reliable source stream —
/// BBR-style `delivery_rate` (uw1cc2, MATRIX-226).
///
/// Every ACK-window heuristic tried on this path was refuted with mechanism
/// (MATRIX-224/225): stall windows under-read, queue-drain ACK clumps
/// over-read (84 MB/s "delivery" on a 25 MB/s link), and clamping converts
/// artifacts into evidence. This sampler measures what actually happened to
/// each packet instead: at send time it snapshots the cumulative delivered
/// counter; at ACK time the rate sample is `Δdelivered / Δwall-time` over
/// the packet's whole flight. A sample can only exceed the true path rate by
/// the counter's retransmit over-count (bounded, and zero on clean links) —
/// ACK batching lengthens the interval and *lowers* the sample rather than
/// spiking it. Packet flights that end in loss/requeue produce NO sample.
///
/// Pure state machine over caller-supplied microsecond timestamps
/// (deterministically lab-tested; the QuicLink adapter feeds it wall micros
/// from `stream_rate_epoch` — NEVER the transport's synthetic app-data
/// clock, which reads ~1 ms RTT on a 50 ms path, MATRIX-225).
struct SourceStreamDeliverySampler {
/// Cumulative acked source-stream bytes (the deduped `acked_bytes` the
/// ACK path already computes; includes spuriously re-sent copies acked
/// under fresh packet numbers — a bounded over-count, zero when clean).
delivered_bytes: u64,
/// When `delivered_bytes` last advanced (or the epoch of the current
/// send burst). The BBR ack-interval bound: a sample's divisor is
/// `max(flight time, now − snapshot of this)`, so a packet sent just
/// before an ACK clump cannot claim the whole clump's bytes over its
/// short flight (that over-read is the last surviving relative of the
/// MATRIX-224 window-aggregate spike).
delivered_time_micros: u64,
/// Send-time metadata per in-flight packet number.
in_flight: BTreeMap<u64, PacketDeliveryMeta>,
/// Wall RTprop: minimum send→ACK-processed interval ever observed.
rtprop_min_micros: Option<u64>,
/// Max sample (bytes/s) from NOT-app-limited flights since `take_window`.
window_max_bps: u64,
/// Max sample (bytes/s) from app-limited flights since `take_window`.
window_max_app_limited_bps: u64,
}
#[derive(Debug, Clone, Copy)]
struct PacketDeliveryMeta {
sent_at_micros: u64,
delivered_snapshot: u64,
delivered_time_snapshot_micros: u64,
/// The sender had no further stream data queued when this packet was
/// built: its flight measures the application, not the path. Per BBR,
/// such samples may only RAISE the bottleneck estimate, never define it
/// downward (a chronically limited sender must not decay its own
/// capacity estimate into the MATRIX-204-era rate-collapse spiral).
app_limited: bool,
}
/// One folded delivery window from the sampler.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
struct DeliveryWindow {
/// Best not-app-limited sample, if any flight completed clean.
max_bps: Option<u64>,
/// Best app-limited sample (only usable to raise the estimate).
max_app_limited_bps: Option<u64>,
}
impl SourceStreamDeliverySampler {
fn new() -> Self {
Self {
delivered_bytes: 0,
delivered_time_micros: 0,
in_flight: BTreeMap::new(),
rtprop_min_micros: None,
window_max_bps: 0,
window_max_app_limited_bps: 0,
}
}
fn on_packet_sent(&mut self, packet_number: u64, now_micros: u64, app_limited: bool) {
// A send burst starting from an empty in-flight set re-bases the
// delivery clock: idle gaps (handshake, admission stalls) must not
// stretch the first samples' ack-interval bound.
if self.in_flight.is_empty() {
self.delivered_time_micros = now_micros;
}
self.in_flight.insert(
packet_number,
PacketDeliveryMeta {
sent_at_micros: now_micros,
delivered_snapshot: self.delivered_bytes,
delivered_time_snapshot_micros: self.delivered_time_micros,
app_limited,
},
);
}
/// Fold newly acknowledged packets: `acked_bytes` is the deduped payload
/// total the caller computed for exactly `packet_numbers`.
fn on_packets_acked(&mut self, packet_numbers: &[u64], acked_bytes: u64, now_micros: u64) {
self.delivered_bytes = self.delivered_bytes.saturating_add(acked_bytes);
for packet_number in packet_numbers {
let Some(meta) = self.in_flight.remove(packet_number) else {
continue;
};
let flight_micros = now_micros.saturating_sub(meta.sent_at_micros).max(1);
// BBR rate-sample divisor: the LONGER of the flight time and the
// delivered-clock interval, so clump bytes can never be claimed
// over a shorter span than they actually took to deliver.
let ack_interval = now_micros
.saturating_sub(meta.delivered_time_snapshot_micros)
.max(1);
let interval = flight_micros.max(ack_interval);
let delta = self.delivered_bytes.saturating_sub(meta.delivered_snapshot);
let sample_bps = delta
.saturating_mul(1_000_000)
.checked_div(interval)
.unwrap_or(0);
// RTprop is the flight time alone (send → ACK-processed).
self.rtprop_min_micros = Some(
self.rtprop_min_micros
.map_or(flight_micros, |min| min.min(flight_micros)),
);
if meta.app_limited {
self.window_max_app_limited_bps = self.window_max_app_limited_bps.max(sample_bps);
} else {
self.window_max_bps = self.window_max_bps.max(sample_bps);
}
}
if !packet_numbers.is_empty() && acked_bytes > 0 {
self.delivered_time_micros = now_micros;
}
}
/// Forget a packet whose flight ended without an ACK (declared lost,
/// threshold-requeued, or drained for retransmit): no sample.
fn on_packet_dropped(&mut self, packet_number: u64) {
self.in_flight.remove(&packet_number);
}
/// Drain the best samples accumulated since the last call.
fn take_window(&mut self) -> DeliveryWindow {
let window = DeliveryWindow {
max_bps: (self.window_max_bps > 0).then_some(self.window_max_bps),
max_app_limited_bps: (self.window_max_app_limited_bps > 0)
.then_some(self.window_max_app_limited_bps),
};
self.window_max_bps = 0;
self.window_max_app_limited_bps = 0;
window
}
fn rtprop_min_micros(&self) -> Option<u64> {
self.rtprop_min_micros
}
}
/// Native ATP-QUIC data-plane send authority.
///
/// QUIC recovery remains useful as ACK/loss/RTT telemetry, but its NewReno cwnd
/// floors at `2*MSS` on random loss. MATRIX-132 showed that this strangles the
/// RaptorQ fountain below rsync on the 50M/bad cell. Spend one token per symbol
/// before handing it to the QUIC DATAGRAM queue; the connection's cwnd is still
/// sampled, but no longer decides whether fountain symbols may leave.
struct NativeDataPlanePacer {
controller: CongestionController,
symbol_frame_bytes: u32,
pacing_rate_bps: u64,
byte_pacer_next_send_at: Option<Instant>,
byte_pacer_burst_bytes: usize,
byte_pacer_burst_remaining: usize,
/// Sleeps taken by `before_send` / `before_send_bytes` (the datagram
/// spray path); folded into the limiter report as `pacing` stalls.
pacing_waits: u64,
pacing_wait_micros: u64,
}
impl NativeDataPlanePacer {
fn new(symbol_frame_len: usize, burst_symbols: usize, rate_bytes_per_sec: u64) -> Self {
let symbol_frame_bytes = u32::try_from(symbol_frame_len.max(1))
.unwrap_or(u32::MAX)
.max(1);
let mut pacer = Self {
controller: CongestionController::new(CongestionConfig::default()),
symbol_frame_bytes,
pacing_rate_bps: rate_bytes_per_sec.max(1),
byte_pacer_next_send_at: None,
byte_pacer_burst_bytes: symbol_frame_len.max(1).saturating_mul(burst_symbols.max(1)),
byte_pacer_burst_remaining: 0,
pacing_waits: 0,
pacing_wait_micros: 0,
};
pacer.controller.configure_for_path_rate(
pacer.pacing_rate_bps.saturating_mul(8).max(1),
pacer.symbol_frame_bytes,
u32::try_from(burst_symbols.max(1)).unwrap_or(u32::MAX),
);
pacer
}
/// Account one pacer sleep of `wait` for the limiter report.
fn note_pacer_wait(&mut self, wait: Duration) {
self.pacing_waits = self.pacing_waits.saturating_add(1);
self.pacing_wait_micros = self
.pacing_wait_micros
.saturating_add(super::duration_to_micros_saturating(wait));
}
fn configure(&mut self, pacing: &QuicSprayPacingDecision) {
self.pacing_rate_bps = pacing.pacing_rate_bps.max(1);
self.byte_pacer_next_send_at = None;
let symbol_frame_bytes = usize::try_from(self.symbol_frame_bytes)
.unwrap_or(usize::MAX)
.max(1);
self.byte_pacer_burst_bytes =
symbol_frame_bytes.saturating_mul(pacing.max_burst_symbols.max(1));
self.byte_pacer_burst_remaining = 0;
self.controller.configure_for_path_rate(
self.pacing_rate_bps.saturating_mul(8).max(1),
self.symbol_frame_bytes,
u32::try_from(pacing.max_burst_symbols.max(1)).unwrap_or(u32::MAX),
);
self.controller.update_congestion_feedback(
data_plane_pacer_rtt(pacing),
pacing.congestion_loss_rate > 0.0,
);
}
fn configure_with_shared_decision(
&mut self,
pacing: &QuicSprayPacingDecision,
shared_decision: Option<DatagramRateDecision>,
) {
self.configure(pacing);
if let Some(decision) = shared_decision {
self.controller.configure_from_rate_decision(
decision,
self.symbol_frame_bytes,
u32::try_from(pacing.max_burst_symbols.max(1)).unwrap_or(u32::MAX),
);
}
}
fn configure_source_stream(&mut self, pacing: &QuicSprayPacingDecision) {
self.configure(pacing);
self.byte_pacer_burst_bytes = self
.byte_pacer_burst_bytes
.max(source_stream_max_frame_bytes())
.max(usize::try_from(QUIC_SOURCE_STREAM_FLUSH_BYTES).unwrap_or(usize::MAX));
self.byte_pacer_burst_remaining = 0;
self.byte_pacer_next_send_at = None;
}
/// Update only the byte-pacer rate (bytes/s), preserving the burst shape.
///
/// Used by the delivery-clocked source-stream controller, which re-rates
/// the pacer every ACK window. Deliberately does NOT reconfigure the
/// token-bucket `controller`: that bucket gates the DATAGRAM
/// (`before_send`) path, and resetting it every ACK window would
/// interfere with FEC repair sends sharing this pacer; the source-stream
/// path (`before_send_bytes`) reads only `pacing_rate_bps`. An already
/// armed deadline is retained so frequent delivery-rate updates cannot
/// create unpaced send-now gaps.
fn set_pacing_rate_bytes_per_s(&mut self, rate_bytes_per_s: u64) {
let rate = rate_bytes_per_s.max(1);
if rate == self.pacing_rate_bps {
return;
}
self.pacing_rate_bps = rate;
}
async fn before_send(
&mut self,
cx: &Cx,
pending_datagrams: usize,
bytes_in_flight: u64,
congestion_window: u64,
) -> Result<(), QuicTransportError> {
loop {
let now = Instant::now();
if self.controller.try_consume_send_budget(now) {
return Ok(());
}
let wait = self.controller.time_until_send_budget(now).clamp(
QUIC_DATA_PLANE_PACER_MIN_PAUSE,
QUIC_DATA_PLANE_PACER_MAX_PAUSE,
);
trace_quic_data_plane_pacer_limited(
cx,
pending_datagrams,
wait,
self.pacing_rate_bps,
bytes_in_flight,
congestion_window,
);
crate::time::sleep(cx.now(), wait).await;
self.note_pacer_wait(wait);
cx.checkpoint().map_err(|_| QuicTransportError::Cancelled)?;
}
}
async fn before_send_bytes(
&mut self,
cx: &Cx,
frame_bytes: usize,
pending_datagrams: usize,
bytes_in_flight: u64,
congestion_window: u64,
) -> Result<(), QuicTransportError> {
while let Some(next_send_at) = self.byte_pacer_next_send_at {
let now = Instant::now();
if now >= next_send_at {
break;
}
let wait = next_send_at.duration_since(now).clamp(
QUIC_DATA_PLANE_PACER_MIN_PAUSE,
QUIC_DATA_PLANE_PACER_MAX_PAUSE,
);
trace_quic_data_plane_pacer_limited(
cx,
pending_datagrams,
wait,
self.pacing_rate_bps,
bytes_in_flight,
congestion_window,
);
crate::time::sleep(cx.now(), wait).await;
self.note_pacer_wait(wait);
cx.checkpoint().map_err(|_| QuicTransportError::Cancelled)?;
}
self.note_bytes_paced(frame_bytes);
Ok(())
}
/// Absolute instant before which the byte pacer holds the next paced send,
/// or `None` when a send may proceed immediately. Read by the source-stream
/// flush loop so it can drain inbound ACKs while the pacer waits
/// (MATRIX-235) instead of sleeping the interval opaquely — the datagram
/// spray path keeps using `before_send_bytes`, which sleeps.
fn byte_pacer_deadline(&self) -> Option<Instant> {
self.byte_pacer_next_send_at
}
/// Advance the byte pacer's burst accounting for the `frame_bytes` just
/// paced, recomputing the next absolute deadline when a burst is exhausted.
/// The wait itself is performed by the caller: `before_send_bytes` for the
/// datagram spray (a plain sleep), or the source-stream flush loop (which
/// drains ACKs during the wait). Splitting the accounting out keeps the
/// pacing RATE and burst shape identical on both paths.
fn note_bytes_paced(&mut self, frame_bytes: usize) {
let now = Instant::now();
let pacer_bytes = frame_bytes.max(1);
if self.byte_pacer_burst_remaining == 0 {
self.byte_pacer_burst_remaining = self.byte_pacer_burst_bytes.max(pacer_bytes);
}
self.byte_pacer_burst_remaining =
self.byte_pacer_burst_remaining.saturating_sub(pacer_bytes);
if self.byte_pacer_burst_remaining == 0 {
let interval = source_stream_pacing_interval(
self.byte_pacer_burst_bytes.max(pacer_bytes),
self.pacing_rate_bps,
);
// Absolute (deadline-credit) schedule with bounded catch-up: the
// next deadline advances from the PREVIOUS deadline, clamped to
// at most one interval of accumulated credit, so between-burst
// pipeline work no longer deflates the effective rate ~20% below
// the setpoint (the gain×efficiency≈1 fixed point that pinned
// clean-large at ~55-63 MB/s). The pure-relative fallback was a
// deliberate pre-flow-control safety: an absolute schedule then
// pushed ramps over the netem queue cliff into multi-round
// retransmit churn (500M 9s→42s; 12.7-41.8s erratic even with
// pop-time coalescing). Its stated revisit conditions — bounded
// receiver-side overshoot and larger drain-per-recovery batches
// — are now structural: the negotiated source-stream window caps
// un-read overshoot at ~2 MiB and the evidence-scaled burst
// drain retires a whole burst loss in 1-2 firings.
let base = self.byte_pacer_next_send_at.map_or(now, |prev| {
prev.max(now.checked_sub(interval).unwrap_or(now))
});
self.byte_pacer_next_send_at = Some(base.checked_add(interval).unwrap_or(now));
}
}
}
fn data_plane_pacer_rtt(pacing: &QuicSprayPacingDecision) -> Option<Duration> {
if pacing.path_rtt_s.is_finite() && pacing.path_rtt_s > 0.0 {
Some(Duration::from_secs_f64(pacing.path_rtt_s))
} else {
None
}
}
fn coalesced_spray_flush_symbol_limit(
pacing_burst_symbols: usize,
max_symbol_frames_per_packet: usize,
max_flush_symbols: usize,
path_loss_rate: f64,
clean_packet_batch_target: usize,
) -> usize {
let flush_cap = max_flush_symbols.max(1);
let packet_width = max_symbol_frames_per_packet.max(1);
let burst = if path_loss_rate < QUIC_CLEAN_SPRAY_MAX_LOSS_RATE {
// Loss-free encrypted sprays are dominated by per-packet QUIC work.
// Queue multiple full protected packets, then sleep by byte count so
// average pacing stays unchanged while UDP GSO has work to batch.
let packet_floor = packet_width
.saturating_mul(clean_packet_batch_target.max(1))
.min(flush_cap);
pacing_burst_symbols
.max(1)
.max(packet_floor)
.max(QUIC_CLEAN_SPRAY_BURST_FLOOR_SYMBOLS)
.min(flush_cap)
} else {
pacing_burst_symbols.max(1).min(flush_cap)
};
let full_packet_multiple = (burst / packet_width).saturating_mul(packet_width);
if full_packet_multiple == 0 {
burst
} else {
full_packet_multiple
}
}
fn quic_clean_spray_coalescing_allowed(pacing: &QuicSprayPacingDecision) -> bool {
QUIC_NATIVE_CLEAN_JUMBO_COALESCING_ENABLED
&& pacing.path_loss_rate < QUIC_CLEAN_SPRAY_MAX_LOSS_RATE
&& pacing.path_rtt_s.is_finite()
&& pacing.path_rtt_s > 0.0
&& pacing.path_rtt_s <= QUIC_CLEAN_COALESCING_MAX_RTT_S
}
fn clean_gso_flush_symbol_cap(
max_spray_symbols_per_flush: usize,
max_symbol_frames_per_packet: usize,
) -> usize {
let base_cap = max_spray_symbols_per_flush.max(1);
let packet_width = max_symbol_frames_per_packet.max(1);
if base_cap < packet_width {
base_cap
} else if base_cap == super::DEFAULT_MAX_SPRAY_SYMBOLS_PER_FLUSH {
base_cap
.saturating_mul(QUIC_CLEAN_GSO_PACKETS_PER_FLUSH)
.min(QUIC_CLEAN_GSO_MAX_FLUSH_SYMBOLS)
} else {
base_cap
}
}
fn spray_handoff_symbol_limit_for(
flush_symbols: usize,
pending_outbound_datagrams: usize,
path_loss_rate: f64,
) -> usize {
let flush_symbols = flush_symbols.max(1);
let remaining = flush_symbols.saturating_sub(pending_outbound_datagrams);
let max_handoff = if path_loss_rate < QUIC_CLEAN_SPRAY_MAX_LOSS_RATE {
flush_symbols
} else {
QUIC_LOSSY_SPRAY_HANDOFF_MAX_SYMBOLS
};
remaining.clamp(1, max_handoff)
}
fn quic_gso_send_strategy(packets: &[OutgoingPacket]) -> UdpSendBatchStrategy {
let gso_segment_bytes = packets
.iter()
.map(|packet| packet.data.len())
.max()
.unwrap_or(1)
.clamp(1, usize::from(u16::MAX));
UdpSendBatchStrategy {
gso_segment_bytes,
max_gso_segments: QUIC_CLEAN_GSO_PACKETS_PER_FLUSH.min(UDP_MAX_GSO_SEGMENTS),
..UdpSendBatchStrategy::default()
}
}
fn quic_varint_len(value: usize) -> usize {
match value {
0..=63 => 1,
64..=16_383 => 2,
16_384..=1_073_741_823 => 4,
_ => 8,
}
}
fn symbol_datagram_frame_len(symbol_size: u16, envelope_header_len: usize) -> usize {
let payload_len = usize::from(symbol_size.max(1)).saturating_add(envelope_header_len);
// ATP DATAGRAM frames are encoded as 0x31 (type + explicit length + payload)
// so many frames can be carried in one protected 1-RTT packet.
1usize
.saturating_add(quic_varint_len(payload_len))
.saturating_add(payload_len)
}
fn inbound_udp_packet_receive_limit(
remaining_datagram_capacity: usize,
max_datagram_frames_per_packet: usize,
) -> usize {
INBOUND_PUMP_BATCH.min(remaining_datagram_capacity / max_datagram_frames_per_packet.max(1))
}
/// An established native QUIC connection plus everything needed to drive its
/// 1-RTT application data plane over a real UDP socket.
pub struct QuicLink {
conn: NativeQuicConnection,
endpoint: QuicUdpEndpoint,
protection: AtpPacketProtection,
peer: SocketAddr,
role: StreamRole,
/// Local short-header packet number fallback for packets not tracked by recovery.
send_pn: u64,
/// Monotonic data-plane clock fed to the connection.
clock: u64,
/// Max application payload that fits one 1-RTT packet under the endpoint MTU.
max_app_payload: usize,
/// Expected upper bound on ATP symbol DATAGRAM frames one received UDP
/// packet may enqueue before the symbol decoder gets a turn to drain them.
max_datagram_frames_per_packet: usize,
/// Auth-posture-aware symbol DATAGRAM frame width used to align paced
/// sender flushes to full protected packets.
max_symbol_frames_per_packet: usize,
/// Current symbol DATAGRAM frames allowed per protected spray packet.
spray_max_datagram_frames_per_packet: usize,
/// Configured upper bound on symbols queued before a protected packet flush.
max_spray_symbols_per_flush: usize,
/// RaptorQ payload bytes represented by one ATP DATAGRAM symbol.
symbol_payload_bytes: u64,
symbol_datagram_frame_len: usize,
data_plane_pacer: NativeDataPlanePacer,
pending_received_packets: VecDeque<ReceivedPacket>,
/// Authenticated + decoded packets parked under receive backpressure.
/// Drained before any raw pending packets so packet order is preserved
/// and AEAD/replay work is never repeated.
pending_decoded_packets: VecDeque<DecodedOneRttPacket>,
/// Delivery-clocked adaptive pacing for the reliable source stream
/// (sender side). `Some` only while a paced source stream is active. The
/// shared controller probes the pacing rate up while ACKed delivery keeps
/// pace and converges toward measured delivery under loss, replacing the
/// fixed `QUIC_RELIABLE_SOURCE_STREAM_MAX_PACING_BPS` setpoint that capped
/// clean encrypted throughput (~37 MB/s) and the collapsed lossy spray
/// rates, without NewReno's mild-loss cwnd collapse (uw1cc2).
stream_rate_controller: Option<SourceStreamRatePacer>,
stream_rate_epoch: Instant,
stream_rate_window_started_micros: u64,
stream_rate_sent_bytes: u64,
stream_rate_acked_bytes: u64,
stream_rate_lost_bytes: u64,
/// Source-stream payload bytes sent but not yet acknowledged (bytes in
/// packets currently tracked by `in_flight_stream_frames`). Gates new
/// data admission so a mis-estimated pacing rate can never run the
/// sender hundreds of MB ahead of the receiver through a loss gap.
stream_unacked_bytes: u64,
/// Per-packet delivered-counter sampler (MATRIX-226): the honest BtlBw/
/// RTprop source for the pacing filter and the BDP admission cap.
stream_delivery_sampler: SourceStreamDeliverySampler,
/// The sender loop has exhausted its source data: subsequent flights are
/// app-limited (tail drain, proof exchange) and may only RAISE the
/// delivery estimate.
stream_source_exhausted: bool,
/// Wall-clock path RTT measured during the QUIC handshake (client side;
/// `None` on the accept side or if no sample landed). The transport RTT
/// estimator is fed by this path's synthetic app-data clock and reads
/// ~1 ms on a 50 ms path (MATRIX-225), so RTprop consumers fall back to
/// this (upper-bound) handshake sample until the sampler has real
/// per-packet minima.
path_rtt_estimate_micros: Option<u64>,
sender_handoff: QuicSenderHandoffStats,
idle_timeout: Duration,
beacons: BeaconScheduler,
pending_control_frames: VecDeque<Frame>,
last_flushed_stream_frames: Vec<SentControlStreamFrame>,
in_flight_stream_frames: BTreeMap<u64, Vec<SentControlStreamFrame>>,
latest_stream_ack_ranges: Vec<NativeAckRange>,
in_flight_datagram_packets: BTreeMap<u64, SentDatagramPacket>,
datagram_bytes_in_flight: u64,
pending_datagram_rate_samples: VecDeque<DatagramRateSample>,
received_source_stream_frames: VecDeque<ReceivedSourceStreamFrame>,
/// Maximum observed `offset + len` on the paced source stream (receiver
/// completion validation).
source_stream_observed_end: u64,
/// Observed FIN end offset on the paced source stream, if any.
source_stream_observed_fin_end: Option<u64>,
paced_source_stream: Option<StreamId>,
/// Bounded send window negotiated for the paced source stream via the
/// HelloAck (`None` when the receiver predates bounded windows). Send
/// credit can never exceed one window, so admission demands are clamped
/// against it.
source_stream_send_window: Option<u64>,
udp_packets_received: u64,
one_rtt_packets_ingested: u64,
non_one_rtt_packets_dropped: u64,
unprotect_packets_dropped: u64,
/// Client only: the final handshake flight (the packets carrying the TLS
/// Finished), retained for post-handshake loss recovery. A TLS 1.3 client
/// completes on *sending* Finished; if that flight is lost the server
/// keeps retransmitting its own long-header flight, which this side's
/// data plane would otherwise just drop as `NotOneRtt` — a mutual wedge
/// until both idle timeouts. When the pump drops a long-header packet
/// while this flight is non-empty, the flight is re-sent (rate-limited)
/// so the server can complete (br-asupersync-jmri58).
final_handshake_flight: Vec<OutgoingPacket>,
/// Rate limiter for `final_handshake_flight` re-sends.
last_final_flight_resend: Option<Instant>,
/// Wall-clock stall threshold for the app-data loss-expiry loops. Doubles
/// on each expiry that declared loss (cap [`APP_LOSS_STALL_PTO_MAX`]),
/// resets to [`SOURCE_STREAM_PTO`] on real ACK progress — see the cap's
/// docs for the spurious-loss wedge this prevents (br-asupersync-daqxbz).
app_loss_stall_pto: Duration,
/// When `app_loss_stall_pto` last doubled. The stall expiry is
/// re-entered on every ~200 ms iteration while the source stream is
/// cwnd-blocked, so without this the threshold doubled per iteration
/// instead of once per interval and saturated within a second.
stall_backoff_at: Option<Instant>,
/// Sender-side limiter telemetry (br-asupersync-bi2462.2).
///
/// Stall reasons with held time, cwnd and in-flight peaks,
/// loss/retransmit counters. Observational only; read out by
/// [`QuicLink::limiter_report`] when the send finishes.
limiter: QuicSendLimiterReport,
/// Sender-gated receive-window growth: the `(send limit, retransmit
/// batches)` marker of the last STREAM_DATA_BLOCKED decision, so the
/// admission gate asks the receiver for more window at most once per
/// window's worth of credit and only when that window was loss-clean
/// (`super::source_stream_window_request_due`).
source_stream_window_request: Option<super::SourceStreamWindowProbe>,
}
#[derive(Debug, Clone, Copy, Default)]
struct QuicSenderHandoffStats {
symbols_queued: u64,
enqueue_micros: u128,
max_pending_before_enqueue: usize,
max_pending_after_enqueue: usize,
queue_full_flushes: u64,
liveness_polls: u64,
liveness_micros: u128,
flushes: u64,
generated_packets: u64,
datagram_frames: u64,
generate_micros: u128,
pacer_wait_micros: u128,
protect_micros: u128,
udp_send_micros: u128,
max_pending_before_flush: usize,
max_datagrams_per_plain_packet: usize,
}
impl QuicSenderHandoffStats {
fn has_symbol_work(&self) -> bool {
self.symbols_queued > 0 || self.flushes > 0 || self.queue_full_flushes > 0
}
fn record_enqueue(
&mut self,
queued: usize,
pending_before: usize,
pending_after: usize,
elapsed: Duration,
) {
self.symbols_queued = self
.symbols_queued
.saturating_add(u64::try_from(queued).unwrap_or(u64::MAX));
self.enqueue_micros = self.enqueue_micros.saturating_add(elapsed.as_micros());
self.max_pending_before_enqueue = self.max_pending_before_enqueue.max(pending_before);
self.max_pending_after_enqueue = self.max_pending_after_enqueue.max(pending_after);
}
fn record_queue_full_flush(&mut self, liveness_elapsed: Duration) {
self.queue_full_flushes = self.queue_full_flushes.saturating_add(1);
self.liveness_polls = self.liveness_polls.saturating_add(1);
self.liveness_micros = self
.liveness_micros
.saturating_add(liveness_elapsed.as_micros());
}
fn record_flush(
&mut self,
packets: usize,
datagram_frames: usize,
pending_before: usize,
max_datagrams_per_plain_packet: usize,
generate_elapsed: Duration,
pacer_wait_elapsed: Duration,
protect_elapsed: Duration,
udp_send_elapsed: Duration,
) {
self.flushes = self.flushes.saturating_add(1);
self.generated_packets = self
.generated_packets
.saturating_add(u64::try_from(packets).unwrap_or(u64::MAX));
self.datagram_frames = self
.datagram_frames
.saturating_add(u64::try_from(datagram_frames).unwrap_or(u64::MAX));
self.generate_micros = self
.generate_micros
.saturating_add(generate_elapsed.as_micros());
self.pacer_wait_micros = self
.pacer_wait_micros
.saturating_add(pacer_wait_elapsed.as_micros());
self.protect_micros = self
.protect_micros
.saturating_add(protect_elapsed.as_micros());
self.udp_send_micros = self
.udp_send_micros
.saturating_add(udp_send_elapsed.as_micros());
self.max_pending_before_flush = self.max_pending_before_flush.max(pending_before);
self.max_datagrams_per_plain_packet = self
.max_datagrams_per_plain_packet
.max(max_datagrams_per_plain_packet);
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
struct NativeReceiveTraceCounters {
udp_packets_received: u64,
one_rtt_packets_ingested: u64,
non_one_rtt_packets_dropped: u64,
unprotect_packets_dropped: u64,
datagrams_received: u64,
datagrams_dropped_on_receive: u64,
pending_datagrams: usize,
pending_received_packets: usize,
inbound_datagram_capacity: usize,
inbound_datagram_available: usize,
inbound_pump_batch_limit: usize,
udp_recv_buffer_requested: Option<usize>,
udp_recv_buffer_applied: Option<usize>,
udp_kernel_rx_queue_bytes: Option<u64>,
udp_kernel_drops: Option<u64>,
}
impl NativeReceiveTraceCounters {
fn capture(link: &QuicLink) -> Self {
let socket = NativeUdpReceiveDiagnostics::capture(&link.endpoint);
Self {
udp_packets_received: link.udp_packets_received,
one_rtt_packets_ingested: link.one_rtt_packets_ingested,
non_one_rtt_packets_dropped: link.non_one_rtt_packets_dropped,
unprotect_packets_dropped: link.unprotect_packets_dropped,
datagrams_received: link.conn.datagrams_received(),
datagrams_dropped_on_receive: link.conn.datagrams_dropped_on_receive(),
pending_datagrams: link.conn.pending_datagram_count(),
pending_received_packets: link.pending_inbound_packets(),
inbound_datagram_capacity: link.conn.inbound_datagram_capacity(),
inbound_datagram_available: link.conn.inbound_datagram_remaining_capacity(),
inbound_pump_batch_limit: INBOUND_PUMP_BATCH,
udp_recv_buffer_requested: socket.recv_buffer_requested,
udp_recv_buffer_applied: socket.recv_buffer_applied,
udp_kernel_rx_queue_bytes: socket.kernel_rx_queue_bytes,
udp_kernel_drops: socket.kernel_drops,
}
}
fn trace_decoded(
&self,
cx: &Cx,
transfer_id: &str,
symbols_accepted: u64,
feedback_rounds: u32,
decode_stats: &super::QuicDecodeStats,
) {
let symbols_accepted_text = symbols_accepted.to_string();
let feedback_rounds_text = feedback_rounds.to_string();
let decode_count_text = decode_stats.decode_count.to_string();
let decode_micros_text = decode_stats.decode_micros.to_string();
let udp_packets_received_text = self.udp_packets_received.to_string();
let one_rtt_packets_ingested_text = self.one_rtt_packets_ingested.to_string();
let non_one_rtt_packets_dropped_text = self.non_one_rtt_packets_dropped.to_string();
let unprotect_packets_dropped_text = self.unprotect_packets_dropped.to_string();
let datagrams_received_text = self.datagrams_received.to_string();
let datagrams_dropped_on_receive_text = self.datagrams_dropped_on_receive.to_string();
let pending_datagrams_text = self.pending_datagrams.to_string();
let pending_received_packets_text = self.pending_received_packets.to_string();
let inbound_datagram_capacity_text = self.inbound_datagram_capacity.to_string();
let inbound_datagram_available_text = self.inbound_datagram_available.to_string();
let inbound_pump_batch_limit_text = self.inbound_pump_batch_limit.to_string();
let udp_recv_buffer_requested_text = option_usize_trace(self.udp_recv_buffer_requested);
let udp_recv_buffer_applied_text = option_usize_trace(self.udp_recv_buffer_applied);
let udp_kernel_rx_queue_bytes_text = option_u64_trace(self.udp_kernel_rx_queue_bytes);
let udp_kernel_drops_text = option_u64_trace(self.udp_kernel_drops);
cx.trace_with_fields(
"atp_quic.receive.decoded",
&[
("symbols_accepted", symbols_accepted_text.as_str()),
("feedback_rounds", feedback_rounds_text.as_str()),
("decode_count", decode_count_text.as_str()),
("decode_micros", decode_micros_text.as_str()),
("datagrams_received", datagrams_received_text.as_str()),
(
"datagrams_dropped_on_receive",
datagrams_dropped_on_receive_text.as_str(),
),
("pending_datagrams", pending_datagrams_text.as_str()),
("reorder_occupancy", pending_datagrams_text.as_str()),
(
"pending_received_packets",
pending_received_packets_text.as_str(),
),
("transfer_id", transfer_id),
],
);
cx.trace_with_fields(
"atp_quic.receive.socket",
&[
("udp_packets_received", udp_packets_received_text.as_str()),
(
"one_rtt_packets_ingested",
one_rtt_packets_ingested_text.as_str(),
),
(
"non_one_rtt_packets_dropped",
non_one_rtt_packets_dropped_text.as_str(),
),
(
"unprotect_packets_dropped",
unprotect_packets_dropped_text.as_str(),
),
(
"inbound_datagram_capacity",
inbound_datagram_capacity_text.as_str(),
),
(
"inbound_datagram_available",
inbound_datagram_available_text.as_str(),
),
(
"inbound_pump_batch_limit",
inbound_pump_batch_limit_text.as_str(),
),
(
"udp_recv_buffer_requested",
udp_recv_buffer_requested_text.as_str(),
),
(
"udp_recv_buffer_applied",
udp_recv_buffer_applied_text.as_str(),
),
(
"udp_kernel_rx_queue_bytes",
udp_kernel_rx_queue_bytes_text.as_str(),
),
("udp_kernel_drops", udp_kernel_drops_text.as_str()),
("transfer_id", transfer_id),
],
);
}
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
struct NativeUdpReceiveDiagnostics {
recv_buffer_requested: Option<usize>,
recv_buffer_applied: Option<usize>,
kernel_rx_queue_bytes: Option<u64>,
kernel_drops: Option<u64>,
}
impl NativeUdpReceiveDiagnostics {
fn capture(endpoint: &QuicUdpEndpoint) -> Self {
let buffer_report = endpoint.buffer_report();
let kernel = linux_udp_proc_receive_stats(endpoint.local_addr());
Self {
recv_buffer_requested: buffer_report.requested_recv_buffer_bytes,
recv_buffer_applied: buffer_report.applied_recv_buffer_bytes,
kernel_rx_queue_bytes: kernel.map(|stats| stats.rx_queue_bytes),
kernel_drops: kernel.map(|stats| stats.drops),
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
struct LinuxUdpProcReceiveStats {
rx_queue_bytes: u64,
drops: u64,
}
fn option_usize_trace(value: Option<usize>) -> String {
value.map_or_else(|| "unknown".to_string(), |value| value.to_string())
}
fn option_u64_trace(value: Option<u64>) -> String {
value.map_or_else(|| "unknown".to_string(), |value| value.to_string())
}
#[cfg(target_os = "linux")]
fn linux_udp_proc_receive_stats(local_addr: SocketAddr) -> Option<LinuxUdpProcReceiveStats> {
std::fs::read_to_string("/proc/net/udp")
.ok()
.and_then(|table| linux_udp_proc_receive_stats_from_table(&table, local_addr))
.or_else(|| {
std::fs::read_to_string("/proc/net/udp6")
.ok()
.and_then(|table| linux_udp_proc_receive_stats_from_table(&table, local_addr))
})
}
#[cfg(not(target_os = "linux"))]
fn linux_udp_proc_receive_stats(_local_addr: SocketAddr) -> Option<LinuxUdpProcReceiveStats> {
None
}
#[cfg(target_os = "linux")]
fn linux_udp_proc_receive_stats_from_table(
table: &str,
local_addr: SocketAddr,
) -> Option<LinuxUdpProcReceiveStats> {
table.lines().skip(1).find_map(|line| {
let fields = line.split_whitespace().collect::<Vec<_>>();
let local = *fields.get(1)?;
let queues = *fields.get(4)?;
if !linux_udp_proc_local_matches(local, local_addr) {
return None;
}
let (_, rx_queue_hex) = queues.split_once(':')?;
let rx_queue_bytes = u64::from_str_radix(rx_queue_hex, 16).ok()?;
let drops = fields.last()?.parse::<u64>().ok()?;
Some(LinuxUdpProcReceiveStats {
rx_queue_bytes,
drops,
})
})
}
#[cfg(target_os = "linux")]
fn linux_udp_proc_local_matches(proc_local: &str, local_addr: SocketAddr) -> bool {
let Some((addr_hex, port_hex)) = proc_local.split_once(':') else {
return false;
};
let Ok(port) = u16::from_str_radix(port_hex, 16) else {
return false;
};
if port != local_addr.port() {
return false;
}
match local_addr.ip() {
IpAddr::V4(ip) => ip.is_unspecified() || parse_linux_udp_proc_ipv4(addr_hex) == Some(ip),
IpAddr::V6(ip) => ip.is_unspecified() || addr_hex.len() == 32,
}
}
#[cfg(target_os = "linux")]
fn parse_linux_udp_proc_ipv4(addr_hex: &str) -> Option<Ipv4Addr> {
if addr_hex.len() != 8 {
return None;
}
let raw = u32::from_str_radix(addr_hex, 16).ok()?;
Some(Ipv4Addr::from(raw.to_le_bytes()))
}
fn native_quic_path_signal_with_observed_loss(
transport: &QuicTransportMachine,
observed_loss: f64,
) -> super::QuicPathSignalSample {
let mut path = super::quic_path_signal_from_transport(transport);
// MATRIX-132: raw QUIC DATAGRAM loss is expected inside the FEC budget. Keep
// NewReno loss/cwnd observable through telemetry, but let the data-plane
// pacer react to sender-observed fountain delivery loss instead.
path.loss_rate = if observed_loss.is_finite() {
observed_loss.clamp(0.0, 0.90)
} else {
0.0
};
path.clamped()
}
impl QuicLink {
fn protection_config() -> AtpPacketProtectionConfig {
AtpPacketProtectionConfig {
// Per-packet proof logging on the data-plane hot path is pure overhead;
// the structured per-frame trace lives in the ATP_QUIC_TRACE hook.
enable_proof_logging: false,
..AtpPacketProtectionConfig::default()
}
}
/// Local socket address (useful for the connecting client to learn its port).
#[must_use]
pub fn local_addr(&self) -> SocketAddr {
self.endpoint.local_addr()
}
fn mark_peer_activity(&mut self) {
self.beacons.mark_peer_activity(Instant::now());
}
fn beacon_measurement(&self) -> BeaconMeasurement {
self.beacons
.latest_rtt()
.map_or_else(BeaconMeasurement::empty, |rtt| {
BeaconMeasurement::with_rtt(u32::try_from(rtt.as_micros()).unwrap_or(u32::MAX), 0)
})
}
fn inbound_receive_packet_limit(&self) -> usize {
inbound_udp_packet_receive_limit(
self.conn.inbound_datagram_remaining_capacity(),
self.max_datagram_frames_per_packet,
)
}
fn queue_received_packets(&mut self, packets: impl IntoIterator<Item = ReceivedPacket>) {
self.pending_received_packets.extend(packets);
}
/// Return not-yet-ingested raw packets to the front of the pending queue,
/// preserving arrival order.
fn requeue_received_packets_front(
&mut self,
packets: impl DoubleEndedIterator<Item = ReceivedPacket>,
) {
for packet in packets.rev() {
self.pending_received_packets.push_front(packet);
}
}
fn take_pending_received_packets(&mut self, limit: usize) -> Vec<ReceivedPacket> {
let take = limit.min(self.pending_received_packets.len());
let mut packets = Vec::with_capacity(take);
for _ in 0..take {
if let Some(packet) = self.pending_received_packets.pop_front() {
packets.push(packet);
}
}
packets
}
fn pending_inbound_packets(&self) -> usize {
self.pending_received_packets
.len()
.saturating_add(self.pending_decoded_packets.len())
}
fn reset_sender_handoff_trace(&mut self) {
self.sender_handoff = QuicSenderHandoffStats::default();
}
fn trace_sender_handoff_summary(&mut self, cx: &Cx, phase: &'static str, symbols: u64) {
let stats = self.sender_handoff;
if !stats.has_symbol_work() {
return;
}
let symbols_s = symbols.to_string();
let symbols_queued_s = stats.symbols_queued.to_string();
let enqueue_micros_s = stats.enqueue_micros.to_string();
let max_pending_before_enqueue_s = stats.max_pending_before_enqueue.to_string();
let max_pending_after_enqueue_s = stats.max_pending_after_enqueue.to_string();
let queue_full_flushes_s = stats.queue_full_flushes.to_string();
let liveness_polls_s = stats.liveness_polls.to_string();
let liveness_micros_s = stats.liveness_micros.to_string();
let flushes_s = stats.flushes.to_string();
let generated_packets_s = stats.generated_packets.to_string();
let datagram_frames_s = stats.datagram_frames.to_string();
let generate_micros_s = stats.generate_micros.to_string();
let pacer_wait_micros_s = stats.pacer_wait_micros.to_string();
let protect_micros_s = stats.protect_micros.to_string();
let udp_send_micros_s = stats.udp_send_micros.to_string();
let max_pending_before_flush_s = stats.max_pending_before_flush.to_string();
let max_datagrams_per_plain_packet_s = stats.max_datagrams_per_plain_packet.to_string();
// Correlation-safe field budget (br-asupersync-an0t8o): LogEntry caps
// at MAX_FIELDS=16, so the old single 18-field entry NEVER recorded
// its tail and lost leading fields to prioritized correlation ids.
// Split: enqueue/liveness accounting here, flush/timing attribution
// on the companion `.flush` entry below — both <=12 explicit fields.
cx.trace_with_fields(
"atp_quic.sender.symbol_handoff_summary",
&[
("phase", phase),
("symbols", symbols_s.as_str()),
("symbols_queued", symbols_queued_s.as_str()),
("enqueue_micros", enqueue_micros_s.as_str()),
(
"max_pending_before_enqueue",
max_pending_before_enqueue_s.as_str(),
),
(
"max_pending_after_enqueue",
max_pending_after_enqueue_s.as_str(),
),
("queue_full_flushes", queue_full_flushes_s.as_str()),
("liveness_polls", liveness_polls_s.as_str()),
("liveness_micros", liveness_micros_s.as_str()),
],
);
cx.trace_with_fields(
"atp_quic.sender.symbol_handoff_flush",
&[
("phase", phase),
("flushes", flushes_s.as_str()),
("generated_packets", generated_packets_s.as_str()),
("datagram_frames", datagram_frames_s.as_str()),
("generate_micros", generate_micros_s.as_str()),
("pacer_wait_micros", pacer_wait_micros_s.as_str()),
("protect_micros", protect_micros_s.as_str()),
("udp_send_micros", udp_send_micros_s.as_str()),
(
"max_pending_before_flush",
max_pending_before_flush_s.as_str(),
),
(
"max_datagrams_per_plain_packet",
max_datagrams_per_plain_packet_s.as_str(),
),
],
);
quic_rqtrace!(
"sender: symbol_handoff_summary phase={} symbols={} symbols_queued={} enqueue_micros={} queue_full_flushes={} liveness_micros={} flushes={} generated_packets={} datagram_frames={} generate_micros={} pacer_wait_micros={} protect_micros={} udp_send_micros={} max_pending_before_enqueue={} max_pending_after_enqueue={} max_pending_before_flush={} max_datagrams_per_plain_packet={}",
phase,
symbols,
stats.symbols_queued,
stats.enqueue_micros,
stats.queue_full_flushes,
stats.liveness_micros,
stats.flushes,
stats.generated_packets,
stats.datagram_frames,
stats.generate_micros,
stats.pacer_wait_micros,
stats.protect_micros,
stats.udp_send_micros,
stats.max_pending_before_enqueue,
stats.max_pending_after_enqueue,
stats.max_pending_before_flush,
stats.max_datagrams_per_plain_packet,
);
self.reset_sender_handoff_trace();
}
fn paced_flush_symbol_limit(&self, pacing: &QuicSprayPacingDecision) -> usize {
let clean_flush_cap = clean_gso_flush_symbol_cap(
self.max_spray_symbols_per_flush,
self.max_symbol_frames_per_packet,
);
let clean_coalescing = quic_clean_spray_coalescing_allowed(pacing);
let max_flush_symbols = if clean_coalescing {
clean_flush_cap
} else {
self.max_spray_symbols_per_flush
};
coalesced_spray_flush_symbol_limit(
pacing.max_burst_symbols,
self.max_symbol_frames_per_packet,
max_flush_symbols,
if clean_coalescing {
pacing.path_loss_rate
} else {
QUIC_CLEAN_SPRAY_MAX_LOSS_RATE
},
QUIC_CLEAN_GSO_PACKETS_PER_FLUSH,
)
}
fn update_spray_packet_coalescing(&mut self, pacing: &QuicSprayPacingDecision) {
self.spray_max_datagram_frames_per_packet = if quic_clean_spray_coalescing_allowed(pacing) {
self.max_symbol_frames_per_packet.max(1)
} else {
1
};
}
fn spray_frame_payload_limit(&self) -> usize {
let frame_budget = self
.symbol_datagram_frame_len
.saturating_mul(self.spray_max_datagram_frames_per_packet.max(1))
.saturating_add(ONE_RTT_COALESCED_CONTROL_HEADROOM);
// `usize::clamp` panics when `min > max`, and this runs on every flush.
// One authenticated symbol DATAGRAM frame can exceed one 1-RTT packet on
// a clean path with a near-`u16::MAX` `symbol_size` (`QuicConfig::validate`
// bounds `max_datagram_size` against `symbol_size`, but never `symbol_size`
// against the packet envelope), which made the old fixed argument order
// `min = symbol_datagram_frame_len > max = max_app_payload` and panicked
// the sender on the first (Hello) flush. Cap the lower bound at the upper
// bound so an over-sized symbol degrades to the packet limit instead.
let max = self.max_app_payload.max(1);
let min = self.symbol_datagram_frame_len.max(1).min(max);
frame_budget.clamp(min, max)
}
async fn service_spray_liveness(
&mut self,
cx: &Cx,
control: &mut NativeQuicFrameTransport,
) -> Result<(), QuicTransportError> {
let _ = self.pump_inbound_for(cx, INBOUND_PUMP_DRAIN_GRACE).await?;
self.service_decoded_spray_liveness(cx, control).await
}
async fn flush_until_outbound_datagrams_drained(
&mut self,
cx: &Cx,
control: &mut NativeQuicFrameTransport,
operation: &'static str,
) -> Result<usize, QuicTransportError> {
let mut total_flushed = 0usize;
let mut last_progress = Instant::now();
loop {
cx.checkpoint().map_err(|_| QuicTransportError::Cancelled)?;
if self.conn.pending_outbound_datagram_count() == 0 {
return Ok(total_flushed);
}
let before = self.conn.pending_outbound_datagram_count();
let flushed = self.flush(cx).await?;
total_flushed = total_flushed.saturating_add(flushed);
if self.conn.pending_outbound_datagram_count() == 0 {
return Ok(total_flushed);
}
let pumped = self.pump_inbound_for(cx, NEEDMORE_PTO).await?;
self.service_decoded_spray_liveness(cx, control).await?;
let after = self.conn.pending_outbound_datagram_count();
if flushed > 0 || pumped > 0 || after < before {
last_progress = Instant::now();
} else {
self.release_expired_data_plane_loss(cx, after)?;
if self.conn.pending_outbound_datagram_count() == 0 {
return Ok(total_flushed);
}
if last_progress.elapsed() >= self.idle_timeout {
return Err(QuicTransportError::Timeout {
operation,
timeout: self.idle_timeout,
});
}
}
}
}
async fn service_decoded_spray_liveness(
&mut self,
cx: &Cx,
control: &mut NativeQuicFrameTransport,
) -> Result<(), QuicTransportError> {
while let Some(frame) = control.try_recv(cx, &mut self.conn)? {
match frame.frame_type() {
FrameType::KeepAlive => self.mark_peer_activity(),
FrameType::ObjectRequest | FrameType::Proof => {
self.mark_peer_activity();
self.pending_control_frames.push_back(frame);
}
got => {
return Err(QuicTransportError::Unexpected {
got,
expected: "KeepAlive | ObjectRequest | Proof while spraying",
});
}
}
}
let measurement = self.beacon_measurement();
if self
.beacons
.next_action(Instant::now(), measurement)
.is_some()
{
send_native_keep_alive(cx, &mut self.conn, control)?;
self.flush(cx).await?;
}
if self.beacons.peer_liveness_expired() {
return Err(QuicTransportError::Timeout {
operation: "spray peer liveness",
timeout: self.idle_timeout,
});
}
Ok(())
}
async fn drop_duplicate_need_more_resends(
&mut self,
cx: &Cx,
control: &mut NativeQuicFrameTransport,
served_need: &QuicNeedMore,
) -> Result<usize, QuicTransportError> {
self.service_spray_liveness(cx, control).await?;
let dropped =
drop_duplicate_need_more_frames(&mut self.pending_control_frames, served_need)?;
if dropped > 0 {
let dropped_text = dropped.to_string();
let queued_text = self.pending_control_frames.len().to_string();
let requested_text = need_more_repair_symbol_count(served_need).to_string();
cx.trace_with_fields(
"atp_quic.sender.drop_duplicate_need_more",
&[
("dropped", dropped_text.as_str()),
("queued_after_drop", queued_text.as_str()),
("requested_repair_symbols", requested_text.as_str()),
],
);
quic_rqtrace!(
"sender: dropped_duplicate_need_more dropped={} queued_after_drop={} requested_repair_symbols={}",
dropped,
self.pending_control_frames.len(),
need_more_repair_symbol_count(served_need),
);
}
Ok(dropped)
}
fn release_expired_data_plane_loss(
&mut self,
cx: &Cx,
pending_datagrams: usize,
) -> Result<(), QuicTransportError> {
let Some(deadline) = self.conn.pto_deadline_micros(cx, self.clock)? else {
return Ok(());
};
if deadline > self.clock {
return Ok(());
}
let event = self.conn.on_loss_timeout_expired(
cx,
PacketNumberSpace::ApplicationData,
self.clock,
)?;
if event.lost_packets > 0 {
trace_quic_data_plane_loss_timeout(
cx,
pending_datagrams,
event.lost_packets,
event.lost_bytes,
self.conn.transport().bytes_in_flight(),
self.conn.transport().congestion_window_bytes(),
self.conn.transport().pto_count(),
);
}
Ok(())
}
fn expire_app_data_loss_timeout(
&mut self,
cx: &Cx,
operation: &'static str,
) -> Result<usize, QuicTransportError> {
let Some(deadline) = self.conn.pto_deadline_micros(cx, self.clock)? else {
// Nothing transport-tracked is in flight, which is the steady
// state of the paced source stream: its recovery is ATP-scoped,
// so the stall threshold must still back off here.
self.back_off_untracked_stall_pto(operation);
return Ok(0);
};
self.clock = self.clock.max(deadline);
let event = self.conn.on_loss_timeout_expired(
cx,
PacketNumberSpace::ApplicationData,
self.clock,
)?;
if event.lost_packets > 0 {
self.limiter.loss_timeouts = self.limiter.loss_timeouts.saturating_add(1);
self.limiter.lost_packets = self
.limiter
.lost_packets
.saturating_add(u64::try_from(event.lost_packets).unwrap_or(u64::MAX));
self.limiter.lost_bytes = self.limiter.lost_bytes.saturating_add(event.lost_bytes);
// This expiry is clock-warped (never waits out real time), so a
// stall loop re-arming it faster than the path RTT declares every
// packet lost before its ACK can return. Back the wall-clock
// stall threshold off exponentially until real ACK progress
// resets it (br-asupersync-daqxbz).
self.double_stall_pto();
quic_rqtrace!(
"sender: app_data_loss_timeout operation={} lost_packets={} lost_bytes={} bytes_in_flight={} congestion_window={} pto_count={} stall_pto_ms={}",
operation,
event.lost_packets,
event.lost_bytes,
self.conn.transport().bytes_in_flight(),
self.conn.transport().congestion_window_bytes(),
self.conn.transport().pto_count(),
self.app_loss_stall_pto.as_millis(),
);
} else {
self.back_off_untracked_stall_pto(operation);
}
Ok(event.lost_packets)
}
/// Back off the wall-clock stall threshold after a stall expiry that the
/// transport did not turn into declared loss.
///
/// The paced source stream and control-priority packets are
/// transport-untracked (`in_flight=false`; their recovery is ATP-scoped
/// through `in_flight_stream_frames`), so their stall expiries never
/// report transport loss and the threshold never backed off: every 200 ms
/// of ACK silence re-sent up to 256 packets, and once the receiver
/// acknowledges per drained batch instead of per packet that burst starves
/// the sender's own ACK feedback and the storm sustains itself (GH#67
/// follow-up). Back off on every expiry that still has ATP-tracked frames
/// outstanding, as RFC 9002 §6.2.1 does for PTO; real ACK progress resets
/// it in `apply_source_stream_ack_ranges`.
fn back_off_untracked_stall_pto(&mut self, operation: &'static str) {
if self.in_flight_stream_frames.is_empty() || !self.double_stall_pto() {
return;
}
quic_rqtrace!(
"sender: app_data_stall_backoff operation={} in_flight_stream_packets={} stall_pto_ms={}",
operation,
self.in_flight_stream_frames.len(),
self.app_loss_stall_pto.as_millis(),
);
}
/// Double the wall-clock stall threshold (capped at
/// [`APP_LOSS_STALL_PTO_MAX`]) at most once per current interval.
///
/// The stall expiry is re-entered on every loop iteration while the
/// source stream stays cwnd-blocked (the callers `continue` without
/// re-arming their retransmit clock), so an unconditional doubling ran
/// 200 -> 400 -> 800 -> 1600 -> 2000 ms within a second and left the
/// 20 s / 45 s wait budgets with a tenth of their retransmit rounds under
/// real loss. Returns whether the threshold changed.
fn double_stall_pto(&mut self) -> bool {
let now = Instant::now();
if let Some(at) = self.stall_backoff_at
&& now.saturating_duration_since(at) < self.app_loss_stall_pto
{
return false;
}
let next = self
.app_loss_stall_pto
.saturating_mul(2)
.min(APP_LOSS_STALL_PTO_MAX);
self.stall_backoff_at = Some(now);
if next == self.app_loss_stall_pto {
return false;
}
self.app_loss_stall_pto = next;
true
}
/// Restore the stall threshold to its base value (real ACK progress or a
/// fresh source stream).
fn reset_stall_pto(&mut self) {
self.app_loss_stall_pto = SOURCE_STREAM_PTO;
self.stall_backoff_at = None;
}
/// Drain all currently-pending application frames, protect each into a 1-RTT
/// packet, and send the batch over UDP. Returns the number of packets sent.
async fn flush(&mut self, cx: &Cx) -> Result<usize, QuicTransportError> {
struct PlainOneRttPacket {
packet_number: u64,
header: [u8; ONE_RTT_HEADER_LEN],
payload: BytesMut,
stream_frames: Vec<SentControlStreamFrame>,
datagram_payload_bytes: u64,
time_sent_micros: u64,
}
let pending_before_flush = self.conn.pending_outbound_datagram_count();
let generate_started = Instant::now();
// Pacer sleeps happen INSIDE the generate loop; account them apart so
// `generate_micros` means CPU/build cost, not throttle time. The
// conflation mis-attributed the clean-large equilibrium to frame
// generation for a whole investigation round (br-asupersync-oh6gm2).
let mut pacer_wait_elapsed = Duration::ZERO;
let mut control_packets_this_flush = 0usize;
let mut plain_packets = Vec::new();
let mut flushed_stream_frames = Vec::new();
let mut datagram_frames = 0usize;
let mut max_datagram_frames_per_plain_packet = 0usize;
let mut plaintext_payload_bytes = 0usize;
loop {
cx.checkpoint().map_err(|_| QuicTransportError::Cancelled)?;
let pending_datagrams = self.conn.pending_outbound_datagram_count();
let admission = if self.role == StreamRole::Client && pending_datagrams > 0 {
self.release_expired_data_plane_loss(cx, pending_datagrams)?;
let admission = data_plane_flush_admission(
self.conn.transport(),
pending_datagrams,
self.spray_frame_payload_limit(),
self.max_app_payload,
);
if let Some(telemetry) = admission.cwnd_telemetry {
trace_quic_data_plane_cwnd_telemetry(cx, pending_datagrams, telemetry);
}
admission
} else {
data_plane_flush_admission(
self.conn.transport(),
pending_datagrams,
self.spray_frame_payload_limit(),
self.max_app_payload,
)
};
// Control-stream priority: the paced bulk branch generates frames
// for the source stream ONLY, so while bulk frames are pending it
// structurally starves never-yet-sent control-stream bytes — a
// tree manifest's tail (>100 control packets) stayed buffered for
// an entire 360s transfer under loss (br-asupersync-daqxbz).
// Control traffic is tiny and latency-critical: drain it first
// (cwnd-exempt, burst-capped per flush), then resume paced bulk.
let control_stream = super::first_client_bidi_stream();
let control_stream_pending = pending_datagrams == 0
&& control_packets_this_flush < QUIC_CONTROL_STREAM_PACKETS_PER_FLUSH
&& self.paced_source_stream != Some(control_stream)
&& self.conn.has_pending_stream_frames_for(control_stream);
let paced_source_stream_pending = !control_stream_pending
&& pending_datagrams == 0
&& self
.paced_source_stream
.is_some_and(|stream| self.conn.has_pending_stream_frames_for(stream));
let max_frame_bytes = if control_stream_pending {
// Control packets bypass the cwnd gate: paced bulk keeps
// bytes_in_flight pinned at the congestion window, so a
// cwnd-gated control drain only ever runs in rare dips and a
// multi-packet manifest never finishes (br-asupersync-daqxbz).
// Control traffic is bounded (a manifest tops out around a
// couple hundred KB) and further rate-limited by the per-flush
// packet cap below.
admission
.max_frame_bytes
.min(source_stream_max_frame_bytes())
} else if paced_source_stream_pending {
admission
.max_frame_bytes
.min(source_stream_max_frame_bytes())
} else if pending_datagrams == 0 && self.conn.has_pending_stream_frames() {
let transport = self.conn.transport();
let available = transport
.congestion_window_bytes()
.saturating_sub(transport.bytes_in_flight());
if available <= QUIC_STREAM_PACKET_OVERHEAD_BUDGET {
break;
}
admission
.max_frame_bytes
.min(source_stream_max_frame_bytes())
.min(
usize::try_from(
available.saturating_sub(QUIC_STREAM_PACKET_OVERHEAD_BUDGET),
)
.unwrap_or(usize::MAX),
)
} else {
admission.max_frame_bytes
};
if max_frame_bytes == 0 {
break;
}
if paced_source_stream_pending {
let source_stream = self
.paced_source_stream
.expect("paced source stream checked above");
let frame_bytes =
usize::try_from(self.conn.pending_stream_data_bytes_for(source_stream))
.unwrap_or(usize::MAX)
.min(max_frame_bytes)
.max(1);
let pacer_wait_started = Instant::now();
let mut pacer_waited = false;
// Pacer wait with concurrent ACK drain (MATRIX-235): while the
// byte pacer holds the source stream below its delivery-clocked
// deadline, pump inbound so ACKs are processed as they arrive.
// This keeps `stream_unacked_bytes` fresh so the downstream
// `wait_source_stream_send_admission` gate no longer serializes
// a whole ACK-drain phase AFTER the pacer idle — the ~40%
// clean-large duty-cycle loss (MATRIX-232). The pacing RATE and
// burst shape are unchanged (the deadline still comes from the
// delivery-clocked schedule; only accounting is split out into
// `note_bytes_paced`), so mild-loss behavior is untouched
// (the MATRIX-202 refutation is not re-tread: no cwnd/rate
// change, just when ACKs are drained relative to the wait).
while let Some(deadline) = self.data_plane_pacer.byte_pacer_deadline() {
let now = Instant::now();
if now >= deadline {
break;
}
pacer_waited = true;
let wait = deadline.duration_since(now).clamp(
QUIC_DATA_PLANE_PACER_MIN_PAUSE,
QUIC_DATA_PLANE_PACER_MAX_PAUSE,
);
trace_quic_data_plane_pacer_limited(
cx,
self.conn.pending_stream_frame_count(),
wait,
self.data_plane_pacer.pacing_rate_bps,
self.conn.transport().bytes_in_flight(),
self.conn.transport().congestion_window_bytes(),
);
self.pump_inbound_for(cx, wait).await?;
cx.checkpoint().map_err(|_| QuicTransportError::Cancelled)?;
}
self.data_plane_pacer.note_bytes_paced(frame_bytes);
let pacer_held = pacer_wait_started.elapsed();
if pacer_waited {
self.note_limiter_stall(QuicSendStallReason::Pacing, pacer_held);
}
pacer_wait_elapsed += pacer_held;
}
let frames = if control_stream_pending {
// Control-only packet: keeps the paced stream's frames out so
// bulk stays pacer-governed while the control stream drains.
self.conn.generate_stream_frames_for(
cx,
PacketNumberSpace::ApplicationData,
control_stream,
max_frame_bytes,
)?
} else if paced_source_stream_pending {
let source_stream = self
.paced_source_stream
.expect("paced source stream checked above");
self.conn.generate_stream_frames_for(
cx,
PacketNumberSpace::ApplicationData,
source_stream,
max_frame_bytes,
)?
} else {
self.conn.generate_frames(
cx,
PacketNumberSpace::ApplicationData,
max_frame_bytes,
)?
};
if frames.is_empty() {
break;
}
if control_stream_pending {
control_packets_this_flush = control_packets_this_flush.saturating_add(1);
}
let mut packet_stream_frames = Vec::new();
for frame in &frames {
if let crate::net::atp::protocol::quic_frames::QuicFrame::Stream {
stream_id,
offset,
data,
..
} = frame
{
let stream_frame = SentControlStreamFrame {
stream: StreamId(stream_id.value()),
offset: offset.map_or(0, |offset| offset.value()),
len: u64::try_from(data.len()).unwrap_or(u64::MAX),
};
if self.paced_source_stream == Some(stream_frame.stream) {
self.stream_rate_sent_bytes =
self.stream_rate_sent_bytes.saturating_add(stream_frame.len);
}
flushed_stream_frames.push(stream_frame);
packet_stream_frames.push(stream_frame);
}
}
let packet_datagram_frames = datagram_frame_count(&frames);
let datagram_payload_bytes = u64::try_from(packet_datagram_frames)
.unwrap_or(u64::MAX)
.saturating_mul(self.symbol_payload_bytes);
let mut payload = BytesMut::new();
NativeQuicConnection::encode_frames(&frames, &mut payload)?;
let packet_len = ONE_RTT_HEADER_LEN
.saturating_add(payload.len())
.saturating_add(ONE_RTT_TAG_LEN);
let packet_number = if self.role == StreamRole::Client {
self.clock = self.clock.saturating_add(CLOCK_STEP_MICROS);
let ack_eliciting = frames.iter().any(frame_is_ack_eliciting_for_recovery);
let uses_paced_recovery =
packet_uses_paced_recovery(&frames, self.paced_source_stream);
// Control-priority packets are transport-untracked: their
// retransmission is ATP-scoped (in_flight_stream_frames +
// threshold requeue + stall PTO), and counting them against a
// cwnd that paced bulk keeps saturated re-starves the control
// stream the priority branch exists to serve — the transport's
// hard cwnd check then fail-closes the whole session
// (br-asupersync-daqxbz). They stay ack-eliciting.
let tracks_quic_recovery = !control_stream_pending
&& packet_tracks_recovery_in_flight(&frames, self.paced_source_stream);
let accounting_bytes = if uses_paced_recovery {
data_plane_packet_accounting_bytes(packet_len)
} else {
u64::try_from(packet_len).unwrap_or(u64::MAX).max(1)
};
let packet_number = if uses_paced_recovery {
self.conn.on_packet_sent(
cx,
PacketNumberSpace::ApplicationData,
accounting_bytes,
ack_eliciting,
false,
self.clock,
)?
} else {
self.conn.on_packet_sent(
cx,
PacketNumberSpace::ApplicationData,
accounting_bytes,
ack_eliciting,
tracks_quic_recovery,
self.clock,
)?
};
self.send_pn = self.send_pn.max(packet_number.saturating_add(1));
packet_number
} else {
let packet_number = self.send_pn;
self.send_pn = self.send_pn.saturating_add(1);
packet_number
};
datagram_frames = datagram_frames.saturating_add(packet_datagram_frames);
max_datagram_frames_per_plain_packet =
max_datagram_frames_per_plain_packet.max(packet_datagram_frames);
plaintext_payload_bytes = plaintext_payload_bytes.saturating_add(payload.len());
let header = encode_one_rtt_header(packet_number);
plain_packets.push(PlainOneRttPacket {
packet_number,
header,
payload,
stream_frames: packet_stream_frames,
datagram_payload_bytes,
time_sent_micros: self.clock,
});
}
let generate_elapsed = generate_started
.elapsed()
.saturating_sub(pacer_wait_elapsed);
let count = plain_packets.len();
if !plain_packets.is_empty() {
let protect_started = Instant::now();
let requests = plain_packets
.iter()
.map(|packet| PacketProtectionRequest {
space: PacketProtectionSpace::OneRtt,
key_phase: false,
packet_number: packet.packet_number,
associated_data: packet.header.as_slice(),
payload: packet.payload.as_ref(),
})
.collect::<Vec<_>>();
let protected_packets =
protection_result(self.protection.protect_packets(cx, &requests))?;
let protect_elapsed = protect_started.elapsed();
let mut packets = Vec::with_capacity(protected_packets.len());
for (plain, protected) in plain_packets.iter().zip(protected_packets) {
debug_assert_eq!(protected.packet_number, plain.packet_number);
let mut data = Vec::with_capacity(
ONE_RTT_HEADER_LEN + protected.ciphertext.len() + ONE_RTT_TAG_LEN,
);
data.extend_from_slice(&plain.header);
data.extend_from_slice(&protected.ciphertext);
data.extend_from_slice(&protected.tag);
if data.len() > ATP_QUIC_UDP_MAX_PACKET {
return Err(QuicTransportError::Quic(format!(
"protected 1-RTT packet too large: {} bytes > {} limit",
data.len(),
ATP_QUIC_UDP_MAX_PACKET
)));
}
packets.push(OutgoingPacket {
dst_addr: self.peer,
data,
send_time: None,
});
}
let send_strategy = quic_gso_send_strategy(&packets);
let udp_send_started = Instant::now();
let report = self
.endpoint
.send_batch_with_strategy(cx, &packets, send_strategy)
.await
.map_err(map_udp_error)?;
let udp_send_elapsed = udp_send_started.elapsed();
if let Some(error) = report.error {
return Err(QuicTransportError::Quic(format!(
"udp endpoint sent {} of {} QUIC packets before error: {error}",
report.packets_processed, count
)));
}
if report.packets_processed != count {
return Err(QuicTransportError::Quic(format!(
"udp endpoint sent {} of {} QUIC packets without reporting an error",
report.packets_processed, count
)));
}
// App-limited marking for the delivery sampler. The classifier
// is END-OF-DATA, set by the sender loop when the source is
// exhausted — NOT "pending queue empty at flush end": the paced
// drive loop drains its queue on essentially every flush, and
// that proxy marked 100 % of flights app-limited on the wire
// (sample_bps=0 for all 1060 windows of a 500M/good rep; the
// filter never learned and the whole build was a behavioral
// no-op — caught by trace sanity, invisible to the lab, which
// models app limitation at the scenario level).
let flush_app_limited = self.stream_source_exhausted;
let sampler_now_micros = self.stream_rate_now_micros();
for plain in &plain_packets {
self.record_sent_datagram_packet(
plain.packet_number,
plain.datagram_payload_bytes,
plain.time_sent_micros,
);
if !plain.stream_frames.is_empty() {
for frame in &plain.stream_frames {
self.stream_unacked_bytes =
self.stream_unacked_bytes.saturating_add(frame.len);
}
self.stream_delivery_sampler.on_packet_sent(
plain.packet_number,
sampler_now_micros,
flush_app_limited,
);
if let Some(previous) = self
.in_flight_stream_frames
.insert(plain.packet_number, plain.stream_frames.clone())
{
for frame in &previous {
self.stream_unacked_bytes =
self.stream_unacked_bytes.saturating_sub(frame.len);
}
}
}
}
trace_quic_flush_coalescing(
cx,
count,
datagram_frames,
max_datagram_frames_per_plain_packet,
self.max_symbol_frames_per_packet,
plaintext_payload_bytes,
report.bytes_processed,
report.native_send_batch_used,
report.gso_send_used,
report.fallback_used,
);
self.sender_handoff.record_flush(
count,
datagram_frames,
pending_before_flush,
max_datagram_frames_per_plain_packet,
generate_elapsed,
pacer_wait_elapsed,
protect_elapsed,
udp_send_elapsed,
);
}
self.last_flushed_stream_frames = flushed_stream_frames;
Ok(count)
}
fn last_flushed_stream_frames(&self) -> Vec<SentControlStreamFrame> {
self.last_flushed_stream_frames.clone()
}
fn stream_frames_still_in_flight(&self, targets: &[SentControlStreamFrame]) -> bool {
self.in_flight_stream_frames
.values()
.flatten()
.any(|frame| {
targets.iter().any(|target| {
if frame.stream != target.stream {
return false;
}
let frame_end = frame.offset.saturating_add(frame.len);
let target_end = target.offset.saturating_add(target.len);
frame.offset < target_end && target.offset < frame_end
})
})
}
fn record_received_source_stream_frames(&mut self, frames: &[QuicFrame]) {
let Some(source_stream) = self.paced_source_stream else {
return;
};
for frame in frames {
let QuicFrame::Stream {
stream_id,
offset,
data,
fin,
} = frame
else {
continue;
};
if stream_id.value() != source_stream.0 || (data.is_empty() && !*fin) {
continue;
}
// Fold into completion-validation scalars instead of retaining a
// payload clone per frame: accumulating `Bytes` clones here held
// the entire transfer in receiver memory (~530 MB on the 500M
// cell) until the one end-of-stream validation pass, which only
// ever needed the maximum observed end offset and the FIN end.
let frame_offset = offset.map_or(0, |offset| offset.value());
let frame_end =
frame_offset.saturating_add(u64::try_from(data.len()).unwrap_or(u64::MAX));
self.source_stream_observed_end = self.source_stream_observed_end.max(frame_end);
if *fin {
self.source_stream_observed_fin_end = Some(frame_end);
}
}
}
/// Only the parked RQ-tap path (`drain_native_source_stream_tap_frames`,
/// itself `dead_code`) consumes payload-carrying frames; the live receive
/// path folds completion evidence into scalars instead of retaining
/// payloads. Reviving the tap path requires re-adding payload capture in
/// `record_received_source_stream_frames`.
#[allow(dead_code)]
fn drain_received_source_stream_frames(&mut self) -> Vec<ReceivedSourceStreamFrame> {
self.received_source_stream_frames.drain(..).collect()
}
fn apply_source_stream_ack_ranges(
&mut self,
cx: &Cx,
acked_ranges: &[NativeAckRange],
) -> Result<(usize, usize), QuicTransportError> {
if acked_ranges.is_empty() || self.in_flight_stream_frames.is_empty() {
return Ok((0, 0));
}
self.latest_stream_ack_ranges = acked_ranges.to_vec();
let before = self.in_flight_stream_frames.len();
let mut acked_bytes = 0u64;
let mut acked_frames: Vec<SentControlStreamFrame> = Vec::new();
let mut acked_packet_numbers: Vec<u64> = Vec::new();
self.in_flight_stream_frames
.retain(|packet_number, frames| {
if packet_in_ack_ranges(*packet_number, acked_ranges) {
for frame in frames.iter() {
acked_bytes = acked_bytes.saturating_add(frame.len);
}
acked_frames.extend_from_slice(frames);
acked_packet_numbers.push(*packet_number);
false
} else {
true
}
});
// Per-packet delivery samples (MATRIX-226): fold the deduped acked
// byte total, then emit one honest rate/RTT sample per acked flight.
self.stream_delivery_sampler.on_packets_acked(
&acked_packet_numbers,
acked_bytes,
self.stream_rate_now_micros(),
);
// Release the acknowledged frames' retained retransmission copies so
// sender memory tracks the un-ACKed window, not the transfer size.
for frame in &acked_frames {
let _ = self
.conn
.release_sent_stream_frame(frame.stream, frame.offset);
}
// Packet-threshold recovery for capped ACK ranges (RFC 9000 §6.1
// spirit): receiver ACK frames carry only the newest
// `MAX_ACK_FRAME_RANGES` SACK ranges, so once a tracked packet number
// falls a reorder-threshold below the largest acknowledged AND outside
// every reported range, it can never be acknowledged again — whether
// its packet was lost or merely rotated out of the reported window.
// Left tracked, such entries pin `stream_unacked_bytes`/cwnd shut for
// the rest of the transfer (the >100-packet tree-manifest burst at
// 10% loss accumulated ~180 ranges and wedged both sides,
// br-asupersync-daqxbz). Requeue their frames now: the copies ride
// fresh packet numbers the next ACK CAN report. Spurious re-sends of
// already-delivered bytes are deduplicated by receiver reassembly and
// are deliberately NOT counted as pacer loss.
let mut threshold_requeued_frames = 0usize;
if let Some(largest_acked) = acked_ranges.iter().map(|range| range.largest).max() {
let cutoff =
largest_acked.saturating_sub(QUIC_SOURCE_STREAM_FAST_RETRANSMIT_PACKET_THRESHOLD);
let stale_packets: Vec<u64> = self
.in_flight_stream_frames
.range(..cutoff)
.map(|(packet_number, _)| *packet_number)
.collect();
if !stale_packets.is_empty() {
let mut stale_frames = Vec::new();
for packet_number in stale_packets {
if let Some(frames) = self.in_flight_stream_frames.remove(&packet_number) {
for frame in &frames {
self.stream_unacked_bytes =
self.stream_unacked_bytes.saturating_sub(frame.len);
}
// Flight ended in a requeue, not an ACK: no sample.
self.stream_delivery_sampler
.on_packet_dropped(packet_number);
stale_frames.extend(frames);
}
}
let stale_frames = dedup_stream_frames_for_retransmit(stale_frames);
threshold_requeued_frames = stale_frames.len();
// Requeue in reverse so the push-front pending queue ends in
// ascending offset order (mirrors `retransmit_stream_frames`).
for frame in stale_frames.iter().rev() {
self.conn
.requeue_sent_stream_frame(cx, frame.stream, frame.offset)?;
}
}
}
quic_rqtrace!(
"sender: stream_ack_ranges ranges={} in_flight_packets={} acked_bytes={} threshold_requeued_frames={}",
acked_ranges.len(),
before,
acked_bytes,
threshold_requeued_frames,
);
if acked_bytes > 0 {
self.stream_unacked_bytes = self.stream_unacked_bytes.saturating_sub(acked_bytes);
self.stream_rate_acked_bytes = self.stream_rate_acked_bytes.saturating_add(acked_bytes);
self.maybe_update_source_stream_rate();
// Real ACK progress: the loss-expiry cadence is honest again.
self.reset_stall_pto();
}
let removed = before.saturating_sub(self.in_flight_stream_frames.len());
Ok((removed, threshold_requeued_frames))
}
/// Arm delivery-clocked adaptive pacing for an active paced source stream,
/// seeded from the current pacing decision's rate.
fn begin_source_stream_rate_control(&mut self, initial_pacing_bytes_per_s: u64) {
// A saturated stall threshold from the previous entry must not
// govern the first loss of a fresh stream.
self.reset_stall_pto();
self.stream_rate_controller = Some(SourceStreamRatePacer::new(initial_pacing_bytes_per_s));
self.stream_rate_window_started_micros = self.stream_rate_now_micros();
self.stream_rate_sent_bytes = 0;
self.stream_rate_acked_bytes = 0;
self.stream_rate_lost_bytes = 0;
self.stream_unacked_bytes = 0;
self.stream_delivery_sampler = SourceStreamDeliverySampler::new();
self.stream_source_exhausted = false;
}
/// Current delivery-clocked in-flight admission cap for the source
/// stream (see [`source_stream_bdp_admission_cap`]): honest per-packet
/// BtlBw × wall RTprop, falling back to the handshake RTT sample and
/// then to the 16 MiB runaway ceiling.
fn source_stream_unacked_admission_max(&self) -> u64 {
let Some(pacer) = self.stream_rate_controller.as_ref() else {
return QUIC_SOURCE_STREAM_UNACKED_MAX_BYTES;
};
source_stream_bdp_admission_cap(
pacer.bottleneck_bytes_per_s(),
self.stream_delivery_sampler
.rtprop_min_micros()
.or(self.path_rtt_estimate_micros),
)
}
/// Record one limiter stall of `held` behind `reason`, sampling the QUIC
/// transport's in-flight / cwnd / ssthresh at the same instant.
fn note_limiter_stall(&mut self, reason: QuicSendStallReason, held: Duration) {
let transport = self.conn.transport();
let (bytes_in_flight, cwnd, ssthresh) = (
transport.bytes_in_flight(),
transport.congestion_window_bytes(),
transport.ssthresh_bytes(),
);
self.limiter
.observe_transport(bytes_in_flight, cwnd, ssthresh);
self.limiter.note_stall(reason, held);
}
/// The sender's limiter telemetry for the finished transfer.
///
/// The accumulated stalls plus the transport's final cwnd / ssthresh /
/// RTT / PTO state, the datagram pacer's sleeps, the admission cap in
/// force, and the endpoint's socket-buffer report and send-error count.
fn limiter_report(&self) -> QuicSendLimiterReport {
let mut report = self.limiter.clone();
let transport = self.conn.transport();
report.observe_transport(
transport.bytes_in_flight(),
transport.congestion_window_bytes(),
transport.ssthresh_bytes(),
);
report.min_rtt_micros = transport.rtt().min_rtt_micros();
report.smoothed_rtt_micros = transport.rtt().smoothed_rtt_micros();
report.pto_count = transport.pto_count();
report.unacked_admission_cap_bytes = self.source_stream_unacked_admission_max();
report.path_rtprop_micros = self
.stream_delivery_sampler
.rtprop_min_micros()
.or(self.path_rtt_estimate_micros);
// `path_bottleneck_bytes_per_s` and `peak_stream_send_window_bytes`
// are running maxima sampled at the admission gate: the rate
// controller and the paced stream are gone by the time the transfer
// reports, so an end-of-transfer read would only see fallbacks.
report.pacing.count = report
.pacing
.count
.saturating_add(self.data_plane_pacer.pacing_waits);
report.pacing.micros = report
.pacing
.micros
.saturating_add(self.data_plane_pacer.pacing_wait_micros);
report.udp_send_errors = self
.endpoint
.metrics()
.send_errors
.load(std::sync::atomic::Ordering::Relaxed);
let buffers = self.endpoint.buffer_report();
report.requested_send_buffer_bytes = buffers.requested_send_buffer_bytes;
report.applied_send_buffer_bytes = buffers.applied_send_buffer_bytes;
report.requested_recv_buffer_bytes = buffers.requested_recv_buffer_bytes;
report.applied_recv_buffer_bytes = buffers.applied_recv_buffer_bytes;
report
}
fn end_source_stream_rate_control(&mut self) {
self.stream_rate_controller = None;
}
fn stream_rate_now_micros(&self) -> u64 {
u64::try_from(self.stream_rate_epoch.elapsed().as_micros()).unwrap_or(u64::MAX)
}
/// Record source-stream payload bytes queued for retransmission as a loss
/// signal for the adaptive pacer.
fn note_source_stream_retransmit_bytes(&mut self, bytes: u64) {
if self.stream_rate_controller.is_some() {
self.stream_rate_lost_bytes = self.stream_rate_lost_bytes.saturating_add(bytes);
}
}
/// Feed the delivery sampler's window into the rate follower and apply
/// the resulting pacing rate to the data-plane pacer.
///
/// Windows shorter than 25 ms are accumulated further so filter slots
/// aren't burned on sub-RTT slivers; the samples themselves are
/// per-packet delivered-counter measurements (MATRIX-226) and carry
/// their own honest intervals.
fn maybe_update_source_stream_rate(&mut self) {
const STREAM_RATE_MIN_WINDOW_MICROS: u64 = 25_000;
if self.stream_rate_controller.is_none() {
return;
}
let now_micros =
u64::try_from(self.stream_rate_epoch.elapsed().as_micros()).unwrap_or(u64::MAX);
let elapsed = now_micros.saturating_sub(self.stream_rate_window_started_micros);
if elapsed < STREAM_RATE_MIN_WINDOW_MICROS {
return;
}
if self.stream_rate_acked_bytes == 0 && self.stream_rate_lost_bytes == 0 {
return;
}
let acked = std::mem::take(&mut self.stream_rate_acked_bytes);
let lost = std::mem::take(&mut self.stream_rate_lost_bytes);
self.stream_rate_sent_bytes = 0;
self.stream_rate_window_started_micros = now_micros;
let window = self.stream_delivery_sampler.take_window();
let rtprop = self
.stream_delivery_sampler
.rtprop_min_micros()
.or(self.path_rtt_estimate_micros);
let Some(pacer) = self.stream_rate_controller.as_mut() else {
return;
};
let rate = pacer.on_delivery_window(window, lost, now_micros, rtprop);
let cwnd_cap = source_stream_bdp_admission_cap(pacer.bottleneck_bytes_per_s(), rtprop);
quic_rqtrace!(
"sender: stream_rate_update rate_bytes_per_s={} acked={} lost={} window_micros={} unacked={} sample_bps={} sample_app_limited_bps={} rtprop_micros={} cwnd_cap={} cycle_phase={} path_rtt_micros={}",
rate,
acked,
lost,
elapsed,
self.stream_unacked_bytes,
window.max_bps.unwrap_or_default(),
window.max_app_limited_bps.unwrap_or_default(),
rtprop.unwrap_or_default(),
cwnd_cap,
pacer.cycle_phase(),
self.path_rtt_estimate_micros.unwrap_or_default(),
);
self.data_plane_pacer.set_pacing_rate_bytes_per_s(rate);
}
fn datagram_payload_bytes_for_frames(&self, datagram_frames: usize) -> u64 {
u64::try_from(datagram_frames)
.unwrap_or(u64::MAX)
.saturating_mul(self.symbol_payload_bytes.max(1))
}
fn queued_datagram_payload_bytes(&self, extra_symbols: usize) -> u64 {
let queued = self
.conn
.pending_outbound_datagram_count()
.saturating_add(extra_symbols);
self.datagram_payload_bytes_for_frames(queued)
}
fn datagram_bytes_in_flight_with_pending(&self, extra_symbols: usize) -> u64 {
self.datagram_bytes_in_flight
.saturating_add(self.queued_datagram_payload_bytes(extra_symbols))
}
fn record_sent_datagram_packet(
&mut self,
packet_number: u64,
payload_bytes: u64,
time_sent_micros: u64,
) {
if payload_bytes == 0 {
return;
}
if let Some(previous) = self.in_flight_datagram_packets.insert(
packet_number,
SentDatagramPacket {
payload_bytes,
time_sent_micros,
},
) {
self.datagram_bytes_in_flight = self
.datagram_bytes_in_flight
.saturating_sub(previous.payload_bytes);
}
self.datagram_bytes_in_flight = self.datagram_bytes_in_flight.saturating_add(payload_bytes);
}
fn apply_datagram_ack_ranges(&mut self, cx: &Cx, acked_ranges: &[NativeAckRange]) {
if acked_ranges.is_empty() || self.in_flight_datagram_packets.is_empty() {
return;
}
let now_micros = self.clock.max(1);
let mut retained = BTreeMap::new();
let mut acked_bytes = 0u64;
let mut lost_bytes = 0u64;
let mut rtt_micros: Option<u64> = None;
let mut acked_packets = 0usize;
let mut lost_packets = 0usize;
for (packet_number, packet) in std::mem::take(&mut self.in_flight_datagram_packets) {
if packet_in_ack_ranges(packet_number, acked_ranges) {
acked_packets = acked_packets.saturating_add(1);
acked_bytes = acked_bytes.saturating_add(packet.payload_bytes);
self.datagram_bytes_in_flight = self
.datagram_bytes_in_flight
.saturating_sub(packet.payload_bytes);
let sample_rtt = now_micros.saturating_sub(packet.time_sent_micros).max(1);
rtt_micros = Some(rtt_micros.map_or(sample_rtt, |rtt: u64| rtt.min(sample_rtt)));
} else if packet_lost_by_ack_gap(packet_number, acked_ranges) {
lost_packets = lost_packets.saturating_add(1);
lost_bytes = lost_bytes.saturating_add(packet.payload_bytes);
self.datagram_bytes_in_flight = self
.datagram_bytes_in_flight
.saturating_sub(packet.payload_bytes);
} else {
retained.insert(packet_number, packet);
}
}
self.in_flight_datagram_packets = retained;
if acked_bytes == 0 && lost_bytes == 0 {
return;
}
let sent_bytes = acked_bytes.saturating_add(lost_bytes).max(1);
let sample = DatagramRateSample {
now_micros,
sent_bytes,
acked_bytes,
lost_bytes,
bytes_in_flight: self.datagram_bytes_in_flight,
rtt_micros,
receiver_credit_bytes: None,
receiver_window_bytes: None,
};
self.pending_datagram_rate_samples.push_back(sample);
if cx.trace_buffer().is_some() {
let acked_packets_s = acked_packets.to_string();
let lost_packets_s = lost_packets.to_string();
let acked_bytes_s = acked_bytes.to_string();
let lost_bytes_s = lost_bytes.to_string();
let sent_bytes_s = sent_bytes.to_string();
let in_flight_s = self.datagram_bytes_in_flight.to_string();
let rtt_s = rtt_micros
.map(|rtt| rtt.to_string())
.unwrap_or_else(|| "none".to_string());
cx.trace_with_fields(
"atp_quic.datagram.ack_sample",
&[
("acked_packets", acked_packets_s.as_str()),
("lost_packets", lost_packets_s.as_str()),
("sent_bytes", sent_bytes_s.as_str()),
("acked_bytes", acked_bytes_s.as_str()),
("lost_bytes", lost_bytes_s.as_str()),
("bytes_in_flight", in_flight_s.as_str()),
("rtt_micros", rtt_s.as_str()),
],
);
}
}
fn drain_datagram_rate_samples(&mut self) -> Vec<DatagramRateSample> {
self.pending_datagram_rate_samples.drain(..).collect()
}
fn observe_pending_datagram_rate_samples(
&mut self,
cx: &Cx,
config: &QuicConfig,
aimd: &mut NativeQuicAimdPacer,
) -> Option<DatagramRateDecision> {
let mut latest = None;
for sample in self.drain_datagram_rate_samples() {
latest = Some(aimd.observe_datagram_ack_sample(cx, config, sample));
}
latest
}
async fn wait_for_datagram_send_admission(
&mut self,
cx: &Cx,
control: &mut NativeQuicFrameTransport,
config: &QuicConfig,
aimd: &mut NativeQuicAimdPacer,
pacing: &mut QuicSprayPacingDecision,
local_pending_symbols: usize,
payload_bytes: u64,
repair_round: bool,
) -> Result<(), QuicTransportError> {
let mut last_progress = Instant::now();
loop {
cx.checkpoint().map_err(|_| QuicTransportError::Cancelled)?;
if self
.observe_pending_datagram_rate_samples(cx, config, aimd)
.is_some()
{
*pacing = self.spray_pacing_decision(
config,
aimd.cap_bps(),
aimd.shared_decision(),
aimd.observed_loss(),
);
if repair_round {
enforce_native_repair_round_pacing(pacing, self.symbol_datagram_frame_len);
}
self.update_spray_packet_coalescing(pacing);
self.data_plane_pacer
.configure_with_shared_decision(pacing, aimd.shared_decision());
}
let committed_payload_bytes =
self.datagram_bytes_in_flight_with_pending(local_pending_symbols);
if repair_round {
let repair_window_bytes = u64::try_from(pacing.cwnd_share_symbols.max(1))
.unwrap_or(u64::MAX)
.saturating_mul(payload_bytes.max(1));
if committed_payload_bytes.saturating_add(payload_bytes.max(1))
<= repair_window_bytes
{
return Ok(());
}
}
let admission = aimd.datagram_send_admission(
config,
self.clock.max(1),
committed_payload_bytes,
payload_bytes.max(1),
);
if admission.is_send() {
return Ok(());
}
let admission_kind = match admission {
DatagramSendAdmission::Send => "send",
DatagramSendAdmission::Wait { .. } => "wait",
DatagramSendAdmission::CwndBlocked { .. } => "cwnd_blocked",
DatagramSendAdmission::ReceiverWindowBlocked { .. } => "receiver_window_blocked",
};
let committed_s = committed_payload_bytes.to_string();
let in_flight_s = self.datagram_bytes_in_flight.to_string();
let queued_s = self.conn.pending_outbound_datagram_count().to_string();
let local_pending_s = local_pending_symbols.to_string();
let payload_s = payload_bytes.to_string();
cx.trace_with_fields(
"atp_quic.spray.shared_cwnd_blocked",
&[
("admission", admission_kind),
("committed_payload_bytes", committed_s.as_str()),
("datagram_bytes_in_flight", in_flight_s.as_str()),
("queued_datagrams", queued_s.as_str()),
("local_pending_symbols", local_pending_s.as_str()),
("payload_bytes", payload_s.as_str()),
],
);
if let Some(wait_micros) = admission.retry_after_micros() {
let wait = Duration::from_micros(wait_micros).clamp(
QUIC_DATA_PLANE_PACER_MIN_PAUSE,
QUIC_DATA_PLANE_PACER_MAX_PAUSE,
);
crate::time::sleep(cx.now(), wait).await;
self.clock = self.clock.saturating_add(duration_micros_u64(wait).max(1));
let stall_reason = match admission {
DatagramSendAdmission::CwndBlocked { .. } => QuicSendStallReason::Cwnd,
DatagramSendAdmission::ReceiverWindowBlocked { .. } => {
QuicSendStallReason::ReceiverWindow
}
DatagramSendAdmission::Send | DatagramSendAdmission::Wait { .. } => {
QuicSendStallReason::Pacing
}
};
self.note_limiter_stall(stall_reason, wait);
continue;
}
let before = self.datagram_bytes_in_flight;
let flushed = self.flush(cx).await?;
let pumped = self.pump_inbound_for(cx, NEEDMORE_PTO).await?;
self.service_decoded_spray_liveness(cx, control).await?;
quic_rqtrace!(
"sender: admission_blocked kind={admission_kind} datagram_bytes_in_flight={} in_flight_datagram_packets={} committed_payload_bytes={committed_payload_bytes} flushed={flushed} pumped={pumped} since_progress_ms={} {}",
self.datagram_bytes_in_flight,
self.in_flight_datagram_packets.len(),
last_progress.elapsed().as_millis(),
self.wait_diagnostics(),
);
if self
.observe_pending_datagram_rate_samples(cx, config, aimd)
.is_some()
{
*pacing = self.spray_pacing_decision(
config,
aimd.cap_bps(),
aimd.shared_decision(),
aimd.observed_loss(),
);
if repair_round {
enforce_native_repair_round_pacing(pacing, self.symbol_datagram_frame_len);
}
self.update_spray_packet_coalescing(pacing);
self.data_plane_pacer
.configure_with_shared_decision(pacing, aimd.shared_decision());
}
if flushed > 0 || pumped > 0 || self.datagram_bytes_in_flight < before {
last_progress = Instant::now();
continue;
}
if last_progress.elapsed() >= self.idle_timeout {
return Err(QuicTransportError::Timeout {
operation: "quic datagram shared cwnd admission",
timeout: self.idle_timeout,
});
}
crate::time::sleep(cx.now(), QUIC_DATA_PLANE_PACER_MIN_PAUSE).await;
self.clock = self
.clock
.saturating_add(duration_micros_u64(QUIC_DATA_PLANE_PACER_MIN_PAUSE).max(1));
}
}
fn drain_ack_gap_lost_stream_frames_for_retransmit(
&mut self,
acked_ranges: &[NativeAckRange],
) -> Vec<SentControlStreamFrame> {
if acked_ranges.is_empty() || self.in_flight_stream_frames.is_empty() {
return Vec::new();
}
// Evidence-scaled drain cap: spurious packet-threshold detections
// under ACK batching are 1-3 packet singles, so the base cap stays
// small (a FLAT 32-packet cap amplified each false positive into
// seconds of churn, MATRIX-210). A genuinely large detected set —
// more gap-lost packets than the base cap — is a real burst loss
// (e.g. a seed-rate overrun dropping ~0.5 MB in one window), and
// draining it 8 packets per firing stretched recovery across
// dozens of PTO-clocked rounds; scale the cap to the evidence.
let detected = self
.in_flight_stream_frames
.keys()
.filter(|packet_number| packet_lost_by_ack_gap(**packet_number, acked_ranges))
.count();
let cap = if detected > QUIC_SOURCE_STREAM_FAST_RETRANSMIT_MAX_PACKETS {
detected.min(QUIC_SOURCE_STREAM_FAST_RETRANSMIT_BURST_MAX_PACKETS)
} else {
QUIC_SOURCE_STREAM_FAST_RETRANSMIT_MAX_PACKETS
};
let mut retained = BTreeMap::new();
let mut lost = Vec::new();
let mut lost_packets = 0usize;
for (packet_number, frames) in std::mem::take(&mut self.in_flight_stream_frames) {
if lost_packets < cap && packet_lost_by_ack_gap(packet_number, acked_ranges) {
lost_packets = lost_packets.saturating_add(1);
for frame in &frames {
self.stream_unacked_bytes = self.stream_unacked_bytes.saturating_sub(frame.len);
}
self.stream_delivery_sampler
.on_packet_dropped(packet_number);
lost.extend(frames);
} else {
retained.insert(packet_number, frames);
}
}
self.in_flight_stream_frames = retained;
dedup_stream_frames_for_retransmit(lost)
}
fn drain_in_flight_stream_frames_for_retransmit(&mut self) -> Vec<SentControlStreamFrame> {
if self.in_flight_stream_frames.is_empty() {
return Vec::new();
}
let drained = std::mem::take(&mut self.in_flight_stream_frames);
for packet_number in drained.keys() {
self.stream_delivery_sampler
.on_packet_dropped(*packet_number);
}
let frames = drained.into_values().flatten().collect::<Vec<_>>();
for frame in &frames {
self.stream_unacked_bytes = self.stream_unacked_bytes.saturating_sub(frame.len);
}
dedup_stream_frames_for_retransmit(frames)
}
fn drain_limited_in_flight_stream_frames_for_retransmit(
&mut self,
max_packets: usize,
) -> Vec<SentControlStreamFrame> {
if self.in_flight_stream_frames.is_empty() || max_packets == 0 {
return Vec::new();
}
let mut retained = BTreeMap::new();
let mut frames = Vec::new();
let mut drained_packets = 0usize;
for (packet_number, packet_frames) in std::mem::take(&mut self.in_flight_stream_frames) {
if drained_packets < max_packets {
drained_packets = drained_packets.saturating_add(1);
for frame in &packet_frames {
self.stream_unacked_bytes = self.stream_unacked_bytes.saturating_sub(frame.len);
}
self.stream_delivery_sampler
.on_packet_dropped(packet_number);
frames.extend(packet_frames);
} else {
retained.insert(packet_number, packet_frames);
}
}
self.in_flight_stream_frames = retained;
dedup_stream_frames_for_retransmit(frames)
}
fn pending_fountain_feedback_count(&self) -> usize {
queued_fountain_feedback_count(&self.pending_control_frames)
}
fn has_pending_fountain_feedback(&self) -> bool {
self.pending_fountain_feedback_count() > 0
}
async fn retransmit_stream_frames(
&mut self,
cx: &Cx,
frames: &[SentControlStreamFrame],
reason: &'static str,
) -> Result<usize, QuicTransportError> {
if frames.is_empty() {
return Ok(0);
}
let mut lost_bytes = 0u64;
for frame in frames {
lost_bytes = lost_bytes.saturating_add(frame.len);
}
self.note_source_stream_retransmit_bytes(lost_bytes);
self.limiter.retransmit_batches = self.limiter.retransmit_batches.saturating_add(1);
self.limiter.retransmitted_stream_bytes = self
.limiter
.retransmitted_stream_bytes
.saturating_add(lost_bytes);
// Requeue in REVERSE: each requeue pushes to the queue front, so
// iterating the dedup-sorted (ascending) list in reverse leaves the
// pending queue in ascending offset order — which is what lets the
// stream's pop-time retransmit coalescing merge contiguous ranges
// into full-size wire frames (and helps receiver reassembly
// locality).
for frame in frames.iter().rev() {
self.conn
.requeue_sent_stream_frame(cx, frame.stream, frame.offset)?;
}
let retransmitted = self.flush(cx).await?;
let frames_s = frames.len().to_string();
let packets_s = retransmitted.to_string();
cx.trace_with_fields(
"atp_quic.control_stream.retransmit",
&[
("reason", reason),
("stream_frames", frames_s.as_str()),
("packets", packets_s.as_str()),
],
);
quic_rqtrace!(
"sender: stream_retransmit reason={} stream_frames={} packets={}",
reason,
frames.len(),
retransmitted
);
Ok(retransmitted)
}
fn process_decoded_one_rtt_packet(
&mut self,
cx: &Cx,
packet: &DecodedOneRttPacket,
) -> Result<InboundPacketDisposition, QuicTransportError> {
let packet_number = packet.packet_number;
let decoded_frames = &packet.frames;
let required_datagram_slots = datagram_frame_count(decoded_frames);
let available_datagram_slots = self.conn.inbound_datagram_remaining_capacity();
// Under symbol-queue pressure, shed only this packet's DATAGRAM frames
// (RFC 9221 receivers may drop datagrams under resource pressure, and
// ATP's fountain-coded symbols are redundant by design) while the
// ACK/STREAM frames keep flowing. Parking the whole packet instead
// would head-of-line block the reliable source stream behind a full
// symbol queue and stall the transfer.
let shed_datagram_frames =
required_datagram_slots > 0 && available_datagram_slots < required_datagram_slots;
let kept_frames: Vec<QuicFrame>;
let process_frames: &[QuicFrame] = if shed_datagram_frames {
if cx.trace_buffer().is_some() {
let packet_number_text = packet_number.to_string();
let required_text = required_datagram_slots.to_string();
let available_text = available_datagram_slots.to_string();
let pending_datagrams_text = self.conn.pending_datagram_count().to_string();
let pending_received_packets_text = self.pending_inbound_packets().to_string();
cx.trace_with_fields(
"atp_quic.receive.datagram_queue_backpressure",
&[
("reason", "shed_datagram_frames_insufficient_slots"),
("packet_number", packet_number_text.as_str()),
("required_datagram_slots", required_text.as_str()),
("available_datagram_slots", available_text.as_str()),
("pending_datagrams", pending_datagrams_text.as_str()),
(
"pending_received_packets",
pending_received_packets_text.as_str(),
),
],
);
}
kept_frames = decoded_frames
.iter()
.filter(|frame| !matches!(frame, QuicFrame::Datagram { .. }))
.cloned()
.collect();
&kept_frames
} else {
decoded_frames
};
let acked_stream_ranges = acked_packet_ranges_from_frames(process_frames)?;
self.clock = self.clock.saturating_add(CLOCK_STEP_MICROS);
match self.conn.process_packet_frames(
cx,
PacketNumberSpace::ApplicationData,
packet_number,
process_frames,
self.clock,
) {
Ok(()) => {
if shed_datagram_frames {
// The packet itself was received and authenticated; only
// its DATAGRAM payload was shed. Acknowledge it so the
// sender's in-flight/cwnd accounting drains instead of
// jamming on packets we deliberately shed.
self.conn.acknowledge_received_packet(
PacketNumberSpace::ApplicationData,
packet_number,
);
}
self.apply_source_stream_ack_ranges(cx, &acked_stream_ranges)?;
self.apply_datagram_ack_ranges(cx, &acked_stream_ranges);
self.record_received_source_stream_frames(process_frames);
self.one_rtt_packets_ingested = self.one_rtt_packets_ingested.saturating_add(1);
self.mark_peer_activity();
Ok(InboundPacketDisposition::Applied)
}
Err(error) if error.is_stream_reassembly_backpressure() => {
// Parking this packet would prevent a later gap-closing frame
// from making capacity available. Drop without ACK and let
// reliable frames return under a fresh packet number instead.
cx.trace("atp_quic.receive.stream_reassembly_backpressure");
quic_rqtrace!(
"receiver: reassembly_backpressure_drop packet_number={packet_number}"
);
Ok(InboundPacketDisposition::ReassemblyDropped)
}
Err(NativeQuicConnectionError::DatagramReceiveQueueFull { capacity }) => {
if cx.trace_buffer().is_some() {
let packet_number_text = packet_number.to_string();
let capacity_text = capacity.to_string();
let pending_datagrams_text = self.conn.pending_datagram_count().to_string();
let pending_received_packets_text = self.pending_inbound_packets().to_string();
let datagrams_received_text = self.conn.datagrams_received().to_string();
let datagrams_dropped_on_receive_text =
self.conn.datagrams_dropped_on_receive().to_string();
cx.trace_with_fields(
"atp_quic.receive.datagram_queue_backpressure",
&[
("packet_number", packet_number_text.as_str()),
("capacity", capacity_text.as_str()),
("pending_datagrams", pending_datagrams_text.as_str()),
(
"pending_received_packets",
pending_received_packets_text.as_str(),
),
("datagrams_received", datagrams_received_text.as_str()),
(
"datagrams_dropped_on_receive",
datagrams_dropped_on_receive_text.as_str(),
),
],
);
}
self.mark_peer_activity();
Ok(InboundPacketDisposition::DatagramBackpressure)
}
Err(err) => Err(err.into()),
}
}
/// Authenticate and decode one received datagram, decrypting in place and
/// decoding frames exactly once as zero-copy views of the datagram buffer.
fn decode_and_unprotect_one_rtt(
&mut self,
cx: &Cx,
packet: ReceivedPacket,
) -> Result<InboundPacketDecode, QuicTransportError> {
let Some((key_phase, packet_number)) = parse_one_rtt_header(&packet.data) else {
return Ok(InboundPacketDecode::NotOneRtt);
};
let mut data = packet.data;
match self.protection.unprotect_one_rtt_in_place_now(
cx,
key_phase,
packet_number,
&mut data,
ONE_RTT_HEADER_LEN,
) {
Outcome::Ok(plaintext_len) => {
data.truncate(ONE_RTT_HEADER_LEN + plaintext_len);
let plaintext = Bytes::from(data).slice(ONE_RTT_HEADER_LEN..);
let frames = NativeQuicConnection::decode_frames_bytes(&plaintext)?;
Ok(InboundPacketDecode::Decoded(DecodedOneRttPacket {
packet_number,
frames,
}))
}
// Undecryptable / replayed / stray packet: drop it (QUIC semantics).
Outcome::Err(_) | Outcome::Cancelled(_) | Outcome::Panicked(_) => {
Ok(InboundPacketDecode::Dropped)
}
}
}
/// Feed a batch of already-received packets (e.g. 1-RTT data that arrived at
/// the server while it was still completing the handshake) into the connection.
fn ingest_packets(
&mut self,
cx: &Cx,
packets: Vec<ReceivedPacket>,
) -> Result<IngestPacketsReport, QuicTransportError> {
let mut report = IngestPacketsReport::default();
let mut packets = packets.into_iter();
while let Some(packet) = packets.next() {
match self.decode_and_unprotect_one_rtt(cx, packet)? {
InboundPacketDecode::Decoded(decoded) => {
match self.process_decoded_one_rtt_packet(cx, &decoded)? {
InboundPacketDisposition::Applied => {
report.packets_consumed = report.packets_consumed.saturating_add(1);
report.one_rtt_packets_processed =
report.one_rtt_packets_processed.saturating_add(1);
}
InboundPacketDisposition::ReassemblyDropped => {
report.packets_consumed = report.packets_consumed.saturating_add(1);
}
InboundPacketDisposition::DatagramBackpressure => {
// Receive backpressure: park the authenticated packet
// (never re-run AEAD — the replay window would reject
// it) and return the untouched raw remainder to the
// front of the pending queue in arrival order.
self.pending_decoded_packets.push_back(decoded);
self.requeue_received_packets_front(packets);
report.receive_backpressure = true;
break;
}
}
}
InboundPacketDecode::Dropped => {
self.unprotect_packets_dropped =
self.unprotect_packets_dropped.saturating_add(1);
report.packets_consumed = report.packets_consumed.saturating_add(1);
}
InboundPacketDecode::NotOneRtt => {
self.non_one_rtt_packets_dropped =
self.non_one_rtt_packets_dropped.saturating_add(1);
report.packets_consumed = report.packets_consumed.saturating_add(1);
}
}
}
Ok(report)
}
/// Drain packets parked under receive backpressure: decoded packets first
/// (already authenticated), then raw pending packets.
fn ingest_pending_received_packets(
&mut self,
cx: &Cx,
limit: usize,
) -> Result<IngestPacketsReport, QuicTransportError> {
let mut report = IngestPacketsReport::default();
while report.packets_consumed < limit {
let Some(decoded) = self.pending_decoded_packets.pop_front() else {
break;
};
match self.process_decoded_one_rtt_packet(cx, &decoded)? {
InboundPacketDisposition::Applied => {
report.packets_consumed = report.packets_consumed.saturating_add(1);
report.one_rtt_packets_processed =
report.one_rtt_packets_processed.saturating_add(1);
}
InboundPacketDisposition::ReassemblyDropped => {
report.packets_consumed = report.packets_consumed.saturating_add(1);
}
InboundPacketDisposition::DatagramBackpressure => {
self.pending_decoded_packets.push_front(decoded);
report.receive_backpressure = true;
return Ok(report);
}
}
}
let remaining = limit.saturating_sub(report.packets_consumed);
if remaining == 0 || self.pending_received_packets.is_empty() {
return Ok(report);
}
let packets = self.take_pending_received_packets(remaining);
let raw_report = self.ingest_packets(cx, packets)?;
report.packets_consumed = report
.packets_consumed
.saturating_add(raw_report.packets_consumed);
report.one_rtt_packets_processed = report
.one_rtt_packets_processed
.saturating_add(raw_report.one_rtt_packets_processed);
report.receive_backpressure = raw_report.receive_backpressure;
Ok(report)
}
fn trace_datagram_queue_needs_drain(&self, cx: &Cx, packets_processed: usize) {
let pending_datagrams = self.conn.pending_datagram_count();
let datagram_capacity = self.conn.inbound_datagram_capacity();
let remaining_capacity = self.conn.inbound_datagram_remaining_capacity();
let pending_datagrams_text = pending_datagrams.to_string();
let datagram_capacity_text = datagram_capacity.to_string();
let remaining_capacity_text = remaining_capacity.to_string();
let packets_processed_text = packets_processed.to_string();
let max_datagrams_per_packet_text = self.max_datagram_frames_per_packet.to_string();
let pending_received_packets_text = self.pending_inbound_packets().to_string();
cx.trace_with_fields(
"atp_quic.receive.datagram_queue_needs_drain",
&[
("pending_datagrams", pending_datagrams_text.as_str()),
("reorder_occupancy", pending_datagrams_text.as_str()),
("inbound_datagram_capacity", datagram_capacity_text.as_str()),
(
"inbound_datagram_available",
remaining_capacity_text.as_str(),
),
("packets_processed", packets_processed_text.as_str()),
(
"max_datagrams_per_packet",
max_datagrams_per_packet_text.as_str(),
),
(
"pending_received_packets",
pending_received_packets_text.as_str(),
),
],
);
}
/// Receive one batch of UDP packets, unprotect each, and feed the recovered
/// 1-RTT frames into the connection. Waits at most `idle_timeout` for the
/// first *successfully processed* packet, then keeps draining short-quiet
/// full batches until the socket appears empty or the per-turn drain budget
/// is reached. Returns the number of packets successfully processed
/// (undecryptable / non-1-RTT packets are silently dropped, per QUIC, and
/// do not count as progress or consume the idle allowance — `Ok(0)` means
/// the full `idle_timeout` elapsed without one processable packet).
async fn pump_inbound_for(
&mut self,
cx: &Cx,
timeout: Duration,
) -> Result<usize, QuicTransportError> {
self.pump_inbound_for_with_drain_budget(cx, timeout, INBOUND_PUMP_MAX_DRAIN_BATCHES)
.await
}
/// Re-send the retained client Finished flight (rate-limited). Called when
/// the pump drops an inbound long-header packet: post-handshake, that is
/// the server retransmitting its flight because it never received our
/// Finished — without this resend both sides wedge until their idle
/// timeouts (br-asupersync-jmri58).
async fn maybe_resend_final_handshake_flight(
&mut self,
cx: &Cx,
) -> Result<(), QuicTransportError> {
if self.final_handshake_flight.is_empty() {
return Ok(());
}
let due = self
.last_final_flight_resend
.is_none_or(|at| at.elapsed() >= HANDSHAKE_RECOVERY_RESEND_PTO);
if !due {
return Ok(());
}
self.last_final_flight_resend = Some(Instant::now());
self.endpoint
.send_batch(cx, &self.final_handshake_flight)
.await
.map_err(map_udp_error)?;
Ok(())
}
async fn pump_inbound_for_with_drain_budget(
&mut self,
cx: &Cx,
timeout: Duration,
max_drain_batches: usize,
) -> Result<usize, QuicTransportError> {
let mut total_processed = 0usize;
let mut batches = 0usize;
let mut next_timeout = timeout;
let max_drain_batches = max_drain_batches.max(1);
// Zero-progress deadline: callers treat `Ok(0)` as "nothing arrived for
// `timeout`", and several turn it into a fatal idle-timeout error. A
// batch whose packets are all dropped (undecryptable, stale, non-1-RTT)
// must therefore keep waiting out the caller's remaining allowance
// rather than short-circuiting a zero back — otherwise one junk UDP
// packet arriving alone fake-expires a 360s idle timeout seconds into
// a healthy transfer (br-asupersync-u6m3dy: every encrypted broken-cell
// receiver died this way while stream data was still flowing).
let zero_progress_deadline = Instant::now().checked_add(timeout);
let parked_before = self.pending_inbound_packets();
loop {
let receive_limit = self.inbound_receive_packet_limit();
if receive_limit == 0 {
self.trace_datagram_queue_needs_drain(cx, total_processed);
return Ok(total_processed);
}
if self.pending_inbound_packets() > 0 {
let report = self.ingest_pending_received_packets(cx, receive_limit)?;
total_processed = total_processed.saturating_add(report.one_rtt_packets_processed);
if report.receive_backpressure {
self.trace_datagram_queue_needs_drain(cx, total_processed);
return Ok(total_processed);
}
// Ingest is fully synchronous now; yield so co-scheduled peer
// tasks (in-process lab proxies, loopback tests) are not
// starved for a whole drain budget.
crate::runtime::yield_now().await;
continue;
}
let probe_before = rqtrace_fallback_io_probe();
let wait_started = Instant::now();
let received = if next_timeout.is_zero() {
// A zero wait must still poll the socket exactly once.
// `timeout` reports its deadline as already elapsed before
// polling the inner future whenever an ambient timer clock is
// present (every task `Cx` under the runtime carries the wall
// clock), so routing a zero wait through it never touched the
// socket and the receiver's keep-alive drain acknowledged
// nothing under the CLI runtime (GH#67 follow-up review).
match futures_lite::future::poll_once(
self.endpoint.receive_batch(cx, receive_limit),
)
.await
{
Some(Ok(packets)) => packets,
Some(Err(err)) => return Err(map_udp_error(err)),
None => return Ok(total_processed),
}
} else {
match crate::time::timeout(
cx.now(),
next_timeout,
self.endpoint.receive_batch(cx, receive_limit),
)
.await
{
Ok(Ok(packets)) => packets,
Ok(Err(err)) => return Err(map_udp_error(err)),
Err(_elapsed) => {
self.trace_pump_idle(
next_timeout,
wait_started.elapsed(),
total_processed,
parked_before,
probe_before,
);
return Ok(total_processed);
}
}
};
let received_len = received.len();
if received_len == 0 {
return Ok(total_processed);
}
self.udp_packets_received = self
.udp_packets_received
.saturating_add(u64::try_from(received_len).unwrap_or(u64::MAX));
let non_one_rtt_before = self.non_one_rtt_packets_dropped;
let report = self.ingest_packets(cx, received)?;
if self.non_one_rtt_packets_dropped > non_one_rtt_before {
// Dropped long-header packet(s): the peer is still
// handshaking — re-offer our Finished flight so it can finish.
self.maybe_resend_final_handshake_flight(cx).await?;
}
total_processed = total_processed.saturating_add(report.one_rtt_packets_processed);
if report.receive_backpressure {
self.trace_datagram_queue_needs_drain(cx, total_processed);
return Ok(total_processed);
}
// Ingest is fully synchronous now; yield between drained batches
// so co-scheduled peer tasks are not starved for a whole budget.
crate::runtime::yield_now().await;
if total_processed == 0 {
// Every packet so far was dropped by unprotect/decode. Dropped
// packets are not peer progress, so they must not consume the
// idle allowance: re-arm the wait with whatever remains of it
// instead of returning the `Ok(0)` that callers escalate to a
// fatal "transport timeout" (br-asupersync-u6m3dy).
let remaining = zero_progress_deadline
.map(|deadline| deadline.saturating_duration_since(Instant::now()));
match remaining {
Some(remaining) if remaining.is_zero() => return Ok(0),
Some(remaining) => next_timeout = remaining,
// Unreachable in practice (deadline overflow): keep the
// original window so the wait stays bounded.
None => next_timeout = timeout,
}
continue;
}
batches = batches.saturating_add(1);
if received_len < receive_limit {
return Ok(total_processed);
}
if batches >= max_drain_batches {
let batches_s = batches.to_string();
let max_batches_s = max_drain_batches.to_string();
let total_processed_s = total_processed.to_string();
cx.trace_with_fields(
"atp_quic.inbound_pump.drain_budget_exhausted",
&[
("batches", batches_s.as_str()),
("max_batches", max_batches_s.as_str()),
("packets_processed", total_processed_s.as_str()),
],
);
return Ok(total_processed);
}
next_timeout = INBOUND_PUMP_DRAIN_GRACE;
}
}
async fn pump_inbound(&mut self, cx: &Cx) -> Result<usize, QuicTransportError> {
self.pump_inbound_for(cx, self.idle_timeout).await
}
/// One-line socket/transport/driver state for wait diagnostics (GH#67
/// follow-up): what sits unread in the kernel receive queue, what the
/// transport still holds, and what the fallback I/O driver dispatched.
/// Tells "no datagram reached this socket" from "this socket was never
/// re-polled after the datagram arrived".
fn wait_diagnostics(&self) -> String {
let socket = NativeUdpReceiveDiagnostics::capture(&self.endpoint);
let transport = self.conn.transport();
format!(
"role={:?} udp_packets_received={} one_rtt_packets_ingested={} \
kernel_rx_queue_bytes={} kernel_drops={} pending_received_packets={} \
pending_datagrams={} pending_stream_frames={} in_flight_stream_packets={} \
bytes_in_flight={} cwnd={} pto_count={} stall_pto_ms={} {}",
self.role,
self.udp_packets_received,
self.one_rtt_packets_ingested,
option_u64_trace(socket.kernel_rx_queue_bytes),
option_u64_trace(socket.kernel_drops),
self.pending_inbound_packets(),
self.conn.pending_datagram_count(),
self.conn.pending_stream_frame_count(),
self.in_flight_stream_frames.len(),
transport.bytes_in_flight(),
transport.congestion_window_bytes(),
transport.pto_count(),
self.app_loss_stall_pto.as_millis(),
describe_fallback_io_probe(fallback_io_probe_now()),
)
}
/// Trace a socket wait that ran out its timeout (only under
/// `ATP_RQ_TRACE`): how many packets this pump call had already applied
/// from the parked queue before it waited, and the fallback driver deltas
/// over the wait.
fn trace_pump_idle(
&self,
timeout: Duration,
waited: Duration,
processed: usize,
parked_before: usize,
before: FallbackIoProbe,
) {
if timeout < PUMP_IDLE_TRACE_MIN_WAIT || !rqtrace_enabled() {
return;
}
quic_rqtrace!(
"pump_idle timeout_ms={} waited_ms={} processed={} parked_before={} {} {}",
timeout.as_millis(),
waited.as_millis(),
processed,
parked_before,
self.wait_diagnostics(),
describe_fallback_io_delta(fallback_io_probe_now(), before),
);
}
fn symbol_round_timeout(&self, timeout: Duration, symbols_accepted: u64) -> QuicTransportError {
let socket = NativeUdpReceiveDiagnostics::capture(&self.endpoint);
QuicTransportError::Quic(format!(
"transport timeout during receive symbol round after {timeout:?}; \
udp_packets_received={} one_rtt_packets_ingested={} \
non_one_rtt_packets_dropped={} unprotect_packets_dropped={} \
datagrams_received={} datagrams_dropped_on_receive={} \
pending_datagrams={} pending_received_packets={} \
udp_recv_buffer_requested={} udp_recv_buffer_applied={} \
udp_kernel_rx_queue_bytes={} udp_kernel_drops={} \
symbols_accepted={symbols_accepted}",
self.udp_packets_received,
self.one_rtt_packets_ingested,
self.non_one_rtt_packets_dropped,
self.unprotect_packets_dropped,
self.conn.datagrams_received(),
self.conn.datagrams_dropped_on_receive(),
self.conn.pending_datagram_count(),
self.pending_inbound_packets(),
option_usize_trace(socket.recv_buffer_requested),
option_usize_trace(socket.recv_buffer_applied),
option_u64_trace(socket.kernel_rx_queue_bytes),
option_u64_trace(socket.kernel_drops),
))
}
fn spray_pacing_decision(
&self,
config: &QuicConfig,
aimd_cap_bps: Option<u64>,
shared_decision: Option<DatagramRateDecision>,
observed_loss: f64,
) -> QuicSprayPacingDecision {
let mut config = config.clone();
if let Some(cap) = aimd_cap_bps {
config.bwlimit_bps = Some(config.bwlimit_bps.map_or(cap, |existing| existing.min(cap)));
}
let path = native_quic_path_signal_with_observed_loss(self.conn.transport(), observed_loss);
let mut pacing = super::quic_spray_pacing_decision_from_config(&config, path);
NativeQuicAimdPacer::apply_shared_decision_to_pacing(
shared_decision,
&mut pacing,
self.symbol_datagram_frame_len,
&config,
);
pacing
}
fn source_stream_pacing_decision(&self, config: &QuicConfig) -> QuicSprayPacingDecision {
let mut pacing = self.spray_pacing_decision(config, None, None, config.round0_loss_target);
promote_source_stream_pacing(&mut pacing, config, self.symbol_datagram_frame_len);
pacing
}
fn spray_handoff_symbol_limit(&self, pacing: &QuicSprayPacingDecision) -> usize {
spray_handoff_symbol_limit_for(
self.paced_flush_symbol_limit(pacing),
self.conn.pending_outbound_datagram_count(),
pacing.path_loss_rate,
)
}
async fn flush_symbol_queue_until_below_limit(
&mut self,
cx: &Cx,
control: &mut NativeQuicFrameTransport,
config: &QuicConfig,
aimd: &mut NativeQuicAimdPacer,
flush_symbols: usize,
pacing: &mut QuicSprayPacingDecision,
flushes: &mut usize,
flushed_packets: &mut usize,
flush_elapsed: &mut Duration,
liveness_elapsed: &mut Duration,
_pause_elapsed: &mut Duration,
repair_round: bool,
) -> Result<(), QuicTransportError> {
while self.conn.pending_outbound_datagram_count() >= flush_symbols {
cx.checkpoint().map_err(|_| QuicTransportError::Cancelled)?;
let flush_start = Instant::now();
let packets_flushed = self.flush(cx).await?;
*flushed_packets = (*flushed_packets).saturating_add(packets_flushed);
*flush_elapsed = (*flush_elapsed).saturating_add(flush_start.elapsed());
*flushes = (*flushes).saturating_add(1);
let liveness_start = Instant::now();
self.service_spray_liveness(cx, control).await?;
if self
.observe_pending_datagram_rate_samples(cx, config, aimd)
.is_some()
{
*pacing = self.spray_pacing_decision(
config,
aimd.cap_bps(),
aimd.shared_decision(),
aimd.observed_loss(),
);
if repair_round {
enforce_native_repair_round_pacing(pacing, self.symbol_datagram_frame_len);
}
self.update_spray_packet_coalescing(pacing);
self.data_plane_pacer
.configure_with_shared_decision(pacing, aimd.shared_decision());
}
let liveness_poll_elapsed = liveness_start.elapsed();
*liveness_elapsed = (*liveness_elapsed).saturating_add(liveness_poll_elapsed);
self.sender_handoff
.record_queue_full_flush(liveness_poll_elapsed);
if self.has_pending_fountain_feedback() {
return Ok(());
}
}
Ok(())
}
/// Spray a bounded symbol batch, flushing first whenever the paced outbound
/// queue is full. MATRIX-112 showed the encrypted clean path parked mostly
/// in futex/scheduler handoffs; this gives the RQ producer one batched
/// handoff into the QUIC sender pump per flush window and traces the seam.
async fn spray_symbol_batch(
&mut self,
cx: &Cx,
control: &mut NativeQuicFrameTransport,
tag: u64,
symbols: &[NativeQuicSpraySymbol],
config: &QuicConfig,
pacing: &mut QuicSprayPacingDecision,
aimd: &mut NativeQuicAimdPacer,
repair_round: bool,
) -> Result<(), QuicTransportError> {
if symbols.is_empty() {
return Ok(());
}
let trace_start = Instant::now();
if self
.observe_pending_datagram_rate_samples(cx, config, aimd)
.is_some()
{
*pacing = self.spray_pacing_decision(
config,
aimd.cap_bps(),
aimd.shared_decision(),
aimd.observed_loss(),
);
}
if repair_round {
enforce_native_repair_round_pacing(pacing, self.symbol_datagram_frame_len);
}
self.update_spray_packet_coalescing(pacing);
self.data_plane_pacer
.configure_with_shared_decision(pacing, aimd.shared_decision());
let flush_symbols = self.paced_flush_symbol_limit(pacing);
let mut flushes = 0usize;
let mut flushed_packets = 0usize;
let mut flush_elapsed = Duration::ZERO;
let mut liveness_elapsed = Duration::ZERO;
let mut pause_elapsed = Duration::ZERO;
let mut max_pending_before = self.conn.pending_outbound_datagram_count();
if max_pending_before >= flush_symbols {
self.flush_symbol_queue_until_below_limit(
cx,
control,
config,
aimd,
flush_symbols,
pacing,
&mut flushes,
&mut flushed_packets,
&mut flush_elapsed,
&mut liveness_elapsed,
&mut pause_elapsed,
repair_round,
)
.await?;
}
let mut encoded_bytes = 0usize;
let mut queued_total = 0usize;
let mut payloads = Vec::with_capacity(symbols.len());
let symbol_payload_bytes = self.symbol_payload_bytes;
for item in symbols {
self.wait_for_datagram_send_admission(
cx,
control,
config,
aimd,
pacing,
payloads.len(),
symbol_payload_bytes,
repair_round,
)
.await?;
let pending_datagrams = self
.conn
.pending_outbound_datagram_count()
.saturating_add(payloads.len());
let bytes_in_flight = self.datagram_bytes_in_flight_with_pending(payloads.len());
let congestion_window = aimd.shared_decision().map_or_else(
|| self.conn.transport().congestion_window_bytes(),
|decision| decision.inflight_limit_bytes,
);
if repair_round {
self.data_plane_pacer
.before_send_bytes(
cx,
self.symbol_datagram_frame_len,
pending_datagrams,
bytes_in_flight,
congestion_window,
)
.await?;
} else {
self.data_plane_pacer
.before_send(cx, pending_datagrams, bytes_in_flight, congestion_window)
.await?;
}
let payload =
super::native_symbol_datagram(&item.symbol, tag, item.entry, item.auth_tag)?;
encoded_bytes = encoded_bytes.saturating_add(payload.len());
payloads.push(payload);
if repair_round {
let pending_before_enqueue = self.conn.pending_outbound_datagram_count();
max_pending_before = max_pending_before.max(pending_before_enqueue);
let enqueue_start = Instant::now();
let queued = super::send_native_symbol_batch(cx, &mut self.conn, payloads)?;
if queued != 1 {
return Err(QuicTransportError::Quic(format!(
"native QUIC repair symbol batch queued {queued} of 1 payloads",
)));
}
queued_total = queued_total.saturating_add(queued);
let pending_after_enqueue = self.conn.pending_outbound_datagram_count();
self.sender_handoff.record_enqueue(
queued,
pending_before_enqueue,
pending_after_enqueue,
enqueue_start.elapsed(),
);
payloads = Vec::with_capacity(1);
if pending_after_enqueue >= flush_symbols {
self.flush_symbol_queue_until_below_limit(
cx,
control,
config,
aimd,
flush_symbols,
pacing,
&mut flushes,
&mut flushed_packets,
&mut flush_elapsed,
&mut liveness_elapsed,
&mut pause_elapsed,
repair_round,
)
.await?;
}
}
}
let pending_before_enqueue = self.conn.pending_outbound_datagram_count();
max_pending_before = max_pending_before.max(pending_before_enqueue);
if !payloads.is_empty() {
let enqueue_start = Instant::now();
let queued = super::send_native_symbol_batch(cx, &mut self.conn, payloads)?;
queued_total = queued_total.saturating_add(queued);
if queued_total != symbols.len() {
return Err(QuicTransportError::Quic(format!(
"native QUIC symbol batch queued {queued_total} of {} payloads",
symbols.len()
)));
}
let pending_after_enqueue = self.conn.pending_outbound_datagram_count();
self.sender_handoff.record_enqueue(
queued,
pending_before_enqueue,
pending_after_enqueue,
enqueue_start.elapsed(),
);
} else if queued_total != symbols.len() {
return Err(QuicTransportError::Quic(format!(
"native QUIC symbol batch queued {queued_total} of {} payloads",
symbols.len()
)));
}
let pending_after_enqueue = self.conn.pending_outbound_datagram_count();
if pending_after_enqueue >= flush_symbols {
self.flush_symbol_queue_until_below_limit(
cx,
control,
config,
aimd,
flush_symbols,
pacing,
&mut flushes,
&mut flushed_packets,
&mut flush_elapsed,
&mut liveness_elapsed,
&mut pause_elapsed,
repair_round,
)
.await?;
}
trace_quic_symbol_handoff(
cx,
queued_total,
encoded_bytes,
pending_before_enqueue,
pending_after_enqueue,
flush_symbols,
flushed_packets,
pacing.pacing_rate_bps,
);
if std::env::var_os("ATP_RQ_TRACE").is_some() {
let symbols_s = symbols.len().to_string();
let flushes_s = flushes.to_string();
let flushed_packets_s = flushed_packets.to_string();
let flush_limit_s = flush_symbols.to_string();
let max_pending_before_s = max_pending_before.to_string();
let pending_after_s = self.conn.pending_outbound_datagram_count().to_string();
let elapsed_micros_s = trace_start.elapsed().as_micros().to_string();
let flush_micros_s = flush_elapsed.as_micros().to_string();
let liveness_micros_s = liveness_elapsed.as_micros().to_string();
let pause_micros_s = pause_elapsed.as_micros().to_string();
cx.trace_with_fields(
"atp_quic.sender.symbol_handoff_batch",
&[
("symbols", symbols_s.as_str()),
("flushes", flushes_s.as_str()),
("flushed_packets", flushed_packets_s.as_str()),
("flush_symbol_limit", flush_limit_s.as_str()),
("max_pending_before", max_pending_before_s.as_str()),
("pending_after", pending_after_s.as_str()),
("elapsed_micros", elapsed_micros_s.as_str()),
("flush_micros", flush_micros_s.as_str()),
("liveness_micros", liveness_micros_s.as_str()),
("pause_micros", pause_micros_s.as_str()),
],
);
quic_rqtrace!(
"sender-native: symbol_handoff_batch symbols={} flushes={} flushed_packets={} flush_symbol_limit={} max_pending_before={} pending_after={} elapsed_us={} flush_us={} liveness_us={} pause_us={}",
symbols.len(),
flushes,
flushed_packets,
flush_symbols,
max_pending_before,
self.conn.pending_outbound_datagram_count(),
trace_start.elapsed().as_micros(),
flush_elapsed.as_micros(),
liveness_elapsed.as_micros(),
pause_elapsed.as_micros(),
);
}
Ok(())
}
async fn finish_paced_spray_round(
&mut self,
cx: &Cx,
control: &mut NativeQuicFrameTransport,
sent: u64,
pacing: &QuicSprayPacingDecision,
) -> Result<(), QuicTransportError> {
if sent == 0 {
return Ok(());
}
let pending_symbols = self.conn.pending_outbound_datagram_count();
self.flush_until_outbound_datagrams_drained(
cx,
control,
"drain cwnd-gated spray before ObjectComplete",
)
.await?;
let flush_symbol_limit = self.paced_flush_symbol_limit(pacing);
let sent_s = sent.to_string();
let pending_symbols_s = pending_symbols.to_string();
let flush_symbol_limit_s = flush_symbol_limit.to_string();
let max_symbol_frames_per_packet_s = self.max_symbol_frames_per_packet.to_string();
let pause_after_burst_micros = Duration::ZERO.as_micros().to_string();
let pacing_rate_bps = pacing.pacing_rate_bps.to_string();
cx.trace_with_fields(
"atp_quic.sender.final_paced_spray_flush",
&[
("symbols", sent_s.as_str()),
(
"pause_after_burst_micros",
pause_after_burst_micros.as_str(),
),
("pacing_rate_bps", pacing_rate_bps.as_str()),
("pending_symbols", pending_symbols_s.as_str()),
("flush_symbol_limit", flush_symbol_limit_s.as_str()),
(
"max_symbol_frames_per_packet",
max_symbol_frames_per_packet_s.as_str(),
),
],
);
cx.checkpoint().map_err(|_| QuicTransportError::Cancelled)
}
/// Pump inbound until a complete ATP control frame is available on `control`,
/// flushing pending outbound (e.g. ACKs) between attempts. A full idle
/// timeout of silence fails closed.
async fn next_control_frame(
&mut self,
cx: &Cx,
control: &mut NativeQuicFrameTransport,
operation: &'static str,
) -> Result<Frame, QuicTransportError> {
loop {
cx.checkpoint().map_err(|_| QuicTransportError::Cancelled)?;
if let Some(frame) = self.pending_control_frames.pop_front() {
return Ok(frame);
}
if let Some(frame) = control.try_recv(cx, &mut self.conn)? {
return Ok(frame);
}
self.flush(cx).await?;
// `pump_inbound` waits up to a full idle window for the first packet;
// it only returns 0 when that window elapsed with no traffic at all,
// which means the peer has gone silent — fail closed. Any packets
// (>0) loop back to re-check for a now-complete control frame.
if self.pump_inbound(cx).await? == 0 {
super::quic_progress(format_args!(
"control: idle_timeout operation={operation} timeout_ms={} {}",
self.idle_timeout.as_millis(),
self.wait_diagnostics()
));
return Err(QuicTransportError::Timeout {
operation,
timeout: self.idle_timeout,
});
}
}
}
async fn next_control_frame_with_stream_pto(
&mut self,
cx: &Cx,
control: &mut NativeQuicFrameTransport,
operation: &'static str,
frames: &[SentControlStreamFrame],
reason: &'static str,
) -> Result<Frame, QuicTransportError> {
if frames.is_empty() && self.in_flight_stream_frames.is_empty() {
return self.next_control_frame(cx, control, operation).await;
}
let max_attempts = needmore_pto_attempt_budget(self.idle_timeout);
let pto = NEEDMORE_PTO;
let mut attempts = 0u32;
let mut last_retransmit = Instant::now();
let started = Instant::now();
let liveness_cap = self
.idle_timeout
.saturating_mul(PROOF_WAIT_LIVENESS_MULTIPLIER);
loop {
cx.checkpoint().map_err(|_| QuicTransportError::Cancelled)?;
if let Some(frame) = self.pending_control_frames.pop_front() {
return Ok(frame);
}
if let Some(frame) = control.try_recv(cx, &mut self.conn)? {
return Ok(frame);
}
self.flush(cx).await?;
let pumped = self.pump_inbound_for(cx, pto).await?;
attempts = stream_pto_attempts_after_pump(attempts, pumped);
// A peer that stays alive at the QUIC layer (keep-alives, ACKs)
// but never delivers the frame must not hold this session open
// forever: the consecutive-silence budget above resets on every
// packet, so an absolute cap bounds the wait regardless.
if started.elapsed() >= liveness_cap {
super::quic_progress(format_args!(
"control: stream_pto_liveness_cap operation={operation} attempts={attempts} cap_ms={} {}",
liveness_cap.as_millis(),
self.wait_diagnostics()
));
return Err(QuicTransportError::Timeout {
operation,
timeout: liveness_cap,
});
}
if pumped > 0 && last_retransmit.elapsed() < pto {
continue;
}
quic_rqtrace!(
"proof_wait_attempt operation={operation} attempt={attempts} max_attempts={max_attempts} pumped={pumped} {}",
self.wait_diagnostics()
);
if attempts > max_attempts {
super::quic_progress(format_args!(
"control: stream_pto_timeout operation={operation} attempts={attempts} timeout_ms={} {}",
self.idle_timeout.as_millis(),
self.wait_diagnostics()
));
return Err(QuicTransportError::Timeout {
operation,
timeout: self.idle_timeout,
});
}
let retransmit_frames = self.drain_in_flight_stream_frames_for_retransmit();
if retransmit_frames.is_empty() {
last_retransmit = Instant::now();
continue;
}
super::quic_progress(format_args!(
"control: stream_pto_retransmit reason={reason} attempt={attempts} frames={} operation={operation}",
retransmit_frames.len()
));
self.retransmit_stream_frames(cx, &retransmit_frames, reason)
.await?;
last_retransmit = Instant::now();
}
}
async fn wait_for_stream_frames_acked(
&mut self,
cx: &Cx,
frames: &mut Vec<SentControlStreamFrame>,
operation: &'static str,
reason: &'static str,
) -> Result<(), QuicTransportError> {
let deadline = cx.now() + self.idle_timeout;
loop {
cx.checkpoint().map_err(|_| QuicTransportError::Cancelled)?;
if cx.now() >= deadline {
return Err(QuicTransportError::Timeout {
operation,
timeout: self.idle_timeout,
});
}
self.flush(cx).await?;
if frames.is_empty() {
*frames = self.last_flushed_stream_frames();
}
let _ = self.pump_inbound_for(cx, NEEDMORE_PTO).await?;
let target_stream_pending = frames
.first()
.is_some_and(|frame| self.conn.has_pending_stream_frames_for(frame.stream));
if !frames.is_empty()
&& !target_stream_pending
&& !self.stream_frames_still_in_flight(frames)
{
return Ok(());
}
let retransmit_frames = self.drain_in_flight_stream_frames_for_retransmit();
if retransmit_frames.is_empty() {
continue;
}
self.retransmit_stream_frames(cx, &retransmit_frames, reason)
.await?;
}
}
async fn next_control_frame_with_source_stream_recovery(
&mut self,
cx: &Cx,
control: &mut NativeQuicFrameTransport,
operation: &'static str,
) -> Result<Frame, QuicTransportError> {
let started = Instant::now();
let mut last_retransmit = Instant::now();
let mut loops = 0u64;
let mut flush_continues = 0u64;
let mut pump_continues = 0u64;
let mut gap_retransmit_frames = 0u64;
let mut pto_expiries = 0u64;
let mut pto_lost_packets = 0u64;
let mut pto_blocked_pending = 0u64;
let mut pto_retransmit_frames = 0u64;
let mut pto_empty_drains = 0u64;
loop {
cx.checkpoint().map_err(|_| QuicTransportError::Cancelled)?;
if started.elapsed() >= self.idle_timeout {
super::quic_progress(format_args!(
"control: source_stream_proof_timeout operation={operation} loops={loops} \
pto_expiries={pto_expiries} {}",
self.wait_diagnostics()
));
return Err(QuicTransportError::Quic(format!(
"transport timeout during {operation} after {:?}; \
in_flight_stream_packets={} pending_stream_frames={} \
pending_stream_bytes={} pacing_rate_bytes_per_s={} \
udp_packets_received={} one_rtt_packets_ingested={} \
loops={loops} flush_continues={flush_continues} \
pump_continues={pump_continues} \
gap_retransmit_frames={gap_retransmit_frames} \
pto_expiries={pto_expiries} pto_lost_packets={pto_lost_packets} \
pto_blocked_pending={pto_blocked_pending} \
pto_retransmit_frames={pto_retransmit_frames} \
pto_empty_drains={pto_empty_drains} bytes_in_flight={} cwnd={}",
self.idle_timeout,
self.in_flight_stream_frames.len(),
self.conn.pending_stream_frame_count(),
self.conn.pending_stream_data_bytes(),
self.data_plane_pacer.pacing_rate_bps,
self.udp_packets_received,
self.one_rtt_packets_ingested,
self.conn.transport().bytes_in_flight(),
self.conn.transport().congestion_window_bytes(),
)));
}
loops = loops.saturating_add(1);
if let Some(frame) = self.pending_control_frames.pop_front() {
return Ok(frame);
}
if let Some(frame) = control.try_recv(cx, &mut self.conn)? {
return Ok(frame);
}
let flushed = self.flush(cx).await?;
let pumped = self.pump_inbound_for(cx, SOURCE_STREAM_PTO).await?;
if pumped > 0 {
pump_continues = pump_continues.saturating_add(1);
let latest_stream_ack_ranges = self.latest_stream_ack_ranges.clone();
let retransmit_frames =
self.drain_ack_gap_lost_stream_frames_for_retransmit(&latest_stream_ack_ranges);
if !retransmit_frames.is_empty() {
gap_retransmit_frames =
gap_retransmit_frames.saturating_add(retransmit_frames.len() as u64);
self.retransmit_stream_frames(
cx,
&retransmit_frames,
"source_stream_ack_gap_retransmit",
)
.await?;
last_retransmit = Instant::now();
}
continue;
}
if flushed > 0 {
flush_continues = flush_continues.saturating_add(1);
continue;
}
if last_retransmit.elapsed() >= self.app_loss_stall_pto {
// Declare PTO loss BEFORE the pending-frames guard: a
// cwnd-blocked flush leaves requeued retransmit frames
// pending forever, and skipping expiry while frames are
// pending would livelock (in-flight accounting never
// releases, so the cwnd gate never opens and nothing is ever
// sent again — both sides then idle out).
pto_expiries = pto_expiries.saturating_add(1);
pto_lost_packets = pto_lost_packets
.saturating_add(self.expire_app_data_loss_timeout(cx, operation)? as u64);
if self.conn.has_pending_stream_frames() {
pto_blocked_pending = pto_blocked_pending.saturating_add(1);
continue;
}
let retransmit_frames = self.drain_limited_in_flight_stream_frames_for_retransmit(
QUIC_SOURCE_STREAM_PTO_RETRANSMIT_MAX_PACKETS,
);
if !retransmit_frames.is_empty() {
pto_retransmit_frames =
pto_retransmit_frames.saturating_add(retransmit_frames.len() as u64);
super::quic_progress(format_args!(
"control: source_stream_proof_wait_retransmit frames={} operation={operation}",
retransmit_frames.len()
));
self.retransmit_stream_frames(
cx,
&retransmit_frames,
"source_stream_proof_wait_pto",
)
.await?;
last_retransmit = Instant::now();
continue;
}
pto_empty_drains = pto_empty_drains.saturating_add(1);
last_retransmit = Instant::now();
}
}
}
}
/// Bind a `QuicUdpEndpoint` on `local`, tuned for the ATP-over-QUIC handshake.
async fn bind_endpoint(cx: &Cx, local: SocketAddr) -> Result<QuicUdpEndpoint, QuicTransportError> {
let udp_config = QuicUdpEndpointConfig {
max_packet_size: ATP_QUIC_UDP_MAX_PACKET,
socket_recv_buffer_size: Some(ATP_QUIC_UDP_SOCKET_BUFFER),
socket_send_buffer_size: Some(ATP_QUIC_UDP_SOCKET_BUFFER),
// The endpoint batch ceiling governs how many packets a single
// `receive_batch` may drain. It must be the *receiver* drain width, not
// the sender's current paced spray burst: capping the receiver at a tiny
// send threshold starves it on a real link where repairs may arrive in
// bursts. `send_batch` only chunks by this value, so a larger ceiling is
// strictly better for sends.
max_batch_size: INBOUND_PUMP_BATCH,
..QuicUdpEndpointConfig::default()
};
QuicUdpEndpoint::bind(cx, local, udp_config)
.await
.map_err(map_udp_error)
}
/// Unspecified local bind address matching the family of `peer`.
fn unspecified_for(peer: SocketAddr) -> SocketAddr {
match peer.ip() {
IpAddr::V4(_) => SocketAddr::new(IpAddr::V4(Ipv4Addr::UNSPECIFIED), 0),
IpAddr::V6(_) => SocketAddr::new(IpAddr::V6(Ipv6Addr::UNSPECIFIED), 0),
}
}
/// Establish a [`QuicLink`] from a completed handshake driver and its endpoint.
fn link_from_handshake(
cx: &Cx,
mut driver: QuicHandshakeDriver,
endpoint: QuicUdpEndpoint,
peer: SocketAddr,
role: StreamRole,
config: &QuicConfig,
) -> Result<QuicLink, QuicTransportError> {
if !driver.is_complete() || !driver.one_rtt_keys_installed() {
return Err(QuicTransportError::Quic(
"handshake completed without installed 1-RTT keys".to_string(),
));
}
// Let one protected UDP packet carry many RFC 9221 DATAGRAM frames while
// keeping each RaptorQ symbol in its own DATAGRAM frame. This preserves
// per-symbol loss granularity, but avoids the MATRIX-39 one-symbol-per-UDP
// packet ceiling that throttled encrypted transfers to packet-rate speed.
// The no-evict receive queue and inbound pump remain the burst bounds.
// Declared-lossy links cap the build at one MTU so packets never
// IP-fragment into netem loss amplification (br-asupersync-u6m3dy).
let max_app_payload = one_rtt_max_payload_for_udp_packet(udp_packet_cap_for_config(config));
let max_datagram_frame_size = config.max_datagram_size.min(max_app_payload);
// Use the smallest ATP symbol envelope this link may legitimately carry so
// the receive batch bound remains safe for both auth postures.
let symbol_frame_len =
symbol_datagram_frame_len(config.symbol_size, super::ENVELOPE_HEADER_LEN);
let max_datagram_frames_per_packet =
coalesced_datagram_frames_per_packet(max_app_payload, symbol_frame_len);
let send_envelope_header_len = if config.symbol_auth_context.is_some() {
super::AUTH_ENVELOPE_HEADER_LEN
} else {
super::ENVELOPE_HEADER_LEN
};
let send_symbol_frame_len =
symbol_datagram_frame_len(config.symbol_size, send_envelope_header_len);
let max_symbol_frames_per_packet =
coalesced_datagram_frames_per_packet(max_app_payload, send_symbol_frame_len);
let stream_flow_limit = super::quic_native_stream_flow_limit(config);
let conn_config = NativeQuicConnectionConfig {
role,
max_datagram_frame_size,
send_window: stream_flow_limit,
recv_window: stream_flow_limit,
connection_send_limit: stream_flow_limit,
connection_recv_limit: stream_flow_limit,
..NativeQuicConnectionConfig::default()
};
let mut conn = NativeQuicConnection::new(conn_config);
// ATP already owns reliable STREAM ACK/loss recovery through
// `in_flight_stream_frames`; keep exactly one retransmission authority.
conn.set_internal_retransmission_tracking(false)?;
conn.begin_handshake(cx)?;
conn.on_handshake_keys_available(cx)?;
conn.on_1rtt_keys_available(cx)?;
if role == StreamRole::Client {
// The driver verified the server certificate via WebPKI during the real
// handshake; record that so the client may confirm (fail-closed otherwise).
conn.record_verified_server_identity();
}
conn.on_handshake_confirmed(cx)?;
if !conn.can_send_1rtt() {
return Err(QuicTransportError::Quic(
"connection did not reach a 1-RTT-capable established state".to_string(),
));
}
let final_handshake_flight = driver.take_final_flight();
let driver_path_rtt_micros = driver.path_rtt_estimate_micros;
let provider: RustlsQuicCryptoProvider = driver.into_provider();
let protection =
AtpPacketProtection::from_provider(Box::new(provider), QuicLink::protection_config());
Ok(QuicLink {
conn,
endpoint,
protection,
peer,
role,
send_pn: 0,
clock: 0,
max_app_payload,
max_datagram_frames_per_packet,
max_symbol_frames_per_packet,
spray_max_datagram_frames_per_packet: max_symbol_frames_per_packet,
max_spray_symbols_per_flush: config.max_spray_symbols_per_flush.max(1),
symbol_payload_bytes: u64::from(config.symbol_size.max(1)),
symbol_datagram_frame_len: send_symbol_frame_len,
data_plane_pacer: NativeDataPlanePacer::new(
send_symbol_frame_len,
config.max_spray_symbols_per_flush.max(1),
config
.bwlimit_bps
.unwrap_or(super::QUIC_DEFAULT_COLD_START_PACING_BYTES_PER_S as u64),
),
pending_received_packets: VecDeque::new(),
pending_decoded_packets: VecDeque::new(),
stream_rate_controller: None,
stream_rate_epoch: Instant::now(),
stream_rate_window_started_micros: 0,
stream_rate_sent_bytes: 0,
stream_rate_acked_bytes: 0,
stream_rate_lost_bytes: 0,
stream_unacked_bytes: 0,
stream_delivery_sampler: SourceStreamDeliverySampler::new(),
stream_source_exhausted: false,
path_rtt_estimate_micros: driver_path_rtt_micros,
idle_timeout: config.idle_timeout,
beacons: BeaconScheduler::new(1, Instant::now()),
pending_control_frames: VecDeque::new(),
last_flushed_stream_frames: Vec::new(),
in_flight_stream_frames: BTreeMap::new(),
latest_stream_ack_ranges: Vec::new(),
in_flight_datagram_packets: BTreeMap::new(),
datagram_bytes_in_flight: 0,
pending_datagram_rate_samples: VecDeque::new(),
received_source_stream_frames: VecDeque::new(),
source_stream_observed_end: 0,
source_stream_observed_fin_end: None,
paced_source_stream: None,
source_stream_send_window: None,
udp_packets_received: 0,
one_rtt_packets_ingested: 0,
non_one_rtt_packets_dropped: 0,
unprotect_packets_dropped: 0,
final_handshake_flight,
last_final_flight_resend: None,
app_loss_stall_pto: SOURCE_STREAM_PTO,
stall_backoff_at: None,
sender_handoff: QuicSenderHandoffStats::default(),
limiter: QuicSendLimiterReport::default(),
source_stream_window_request: None,
})
}
/// Connect to `addr` as a QUIC client: bind an ephemeral UDP socket, run the real
/// TLS-1.3 handshake (verifying the server identity), and return an established
/// [`QuicLink`].
async fn connect(
cx: &Cx,
addr: SocketAddr,
client_tls: &QuicClientTls,
config: &QuicConfig,
) -> Result<QuicLink, QuicTransportError> {
cx.checkpoint().map_err(|_| QuicTransportError::Cancelled)?;
let mut endpoint = bind_endpoint(cx, unspecified_for(addr)).await?;
let mut driver = QuicHandshakeDriver::client(
client_tls.config.clone(),
client_tls.server_name.clone(),
ATP_QUIC_ALPN.to_vec(),
)
.map_err(map_tls_error)?;
let dcid = ConnectionId::new(ATP_QUIC_INITIAL_DCID)
.map_err(|err| QuicTransportError::Quic(format!("initial dcid: {err}")))?;
let scid = ConnectionId::new(ATP_QUIC_CLIENT_SCID)
.map_err(|err| QuicTransportError::Quic(format!("client scid: {err}")))?;
match crate::time::timeout(
cx.now(),
config.handshake_timeout,
client_handshake_over_udp(cx, &mut endpoint, addr, &mut driver, dcid, scid),
)
.await
{
Ok(Ok(())) => {}
Ok(Err(err)) => return Err(map_tls_error(err)),
Err(_elapsed) => {
return Err(QuicTransportError::Timeout {
operation: "quic client handshake",
timeout: config.handshake_timeout,
});
}
}
link_from_handshake(cx, driver, endpoint, addr, StreamRole::Client, config)
}
/// Bound on handshake flights before giving up (mirrors the driver's own bound).
const HANDSHAKE_MAX_FLIGHTS: usize = 64;
/// Cadence for handshake loss-recovery re-sends outside the driver loops: the
/// server re-offering its flight when early 1-RTT data proves the client
/// finished, and the client re-offering its Finished flight when dropped
/// long-header packets prove the server did not (br-asupersync-jmri58).
const HANDSHAKE_RECOVERY_RESEND_PTO: Duration = Duration::from_millis(750);
/// Per-flush cap on cwnd-exempt control-stream packets. Control drains ahead
/// of paced bulk (see the flush's control-priority branch) but must not blast
/// an entire buffered manifest into one burst on a lossy link.
const QUIC_CONTROL_STREAM_PACKETS_PER_FLUSH: usize = 8;
/// Cap for the exponential backoff of the app-data loss stall threshold.
///
/// `expire_app_data_loss_timeout` advances the link's synthetic clock to the
/// transport PTO deadline and declares loss immediately — it never consults
/// wall time. When a stall loop re-armed that expiry every fixed
/// `SOURCE_STREAM_PTO` (200ms) on a path whose real RTT is longer (the broken
/// netem is ~400-500ms), every in-flight packet was declared lost and
/// front-requeued BEFORE its ACK could physically return: acked bytes pinned
/// at zero, retransmit copies starved never-yet-sent tail frames forever, and
/// tree manifests wedged both sides to their idle timeouts
/// (br-asupersync-daqxbz). The stall threshold now doubles on every expiry
/// that actually declared loss and resets on real ACK progress, so it
/// converges above any real RTT within two rounds while fast links keep the
/// 200ms base.
const APP_LOSS_STALL_PTO_MAX: Duration = Duration::from_secs(2);
/// True if a received UDP packet is a QUIC long-header packet (handshake), vs a
/// short-header 1-RTT data-plane packet. The long-header form bit is `0x80`.
fn is_long_header(data: &[u8]) -> bool {
data.first().is_some_and(|byte| byte & 0x80 != 0)
}
/// Pump the driver's pending outbound handshake bytes and send each as a
/// protected long-header packet to `peer`. OneRtt-level post-handshake segments
/// belong to the data plane and are skipped (the data plane carries its own).
async fn send_server_handshake_flight(
cx: &Cx,
endpoint: &mut QuicUdpEndpoint,
driver: &mut QuicHandshakeDriver,
peer: SocketAddr,
dst_cid: ConnectionId,
src_cid: ConnectionId,
packet_number: &mut u64,
) -> Result<Vec<OutgoingPacket>, QuicTransportError> {
let segments = driver.pump_outbound().map_err(map_tls_error)?;
let mut packets = Vec::new();
for segment in segments {
if segment.level == HandshakeLevel::OneRtt {
continue;
}
let data = driver
.assemble_handshake_packet(&segment, dst_cid, src_cid, *packet_number)
.map_err(map_tls_error)?;
*packet_number = packet_number.saturating_add(1);
packets.push(OutgoingPacket {
dst_addr: peer,
data,
send_time: None,
});
}
if !packets.is_empty() {
endpoint
.send_batch(cx, &packets)
.await
.map_err(map_udp_error)?;
}
Ok(packets)
}
/// Accept one QUIC client on a bound server `endpoint`: run the real TLS-1.3
/// handshake (presenting the server certificate) and return an established
/// [`QuicLink`] plus any 1-RTT data-plane packets that arrived before the
/// handshake completed (the client finishes the handshake first and may start
/// the data plane immediately).
///
/// This drives the accept-side handshake directly (rather than the driver's
/// `server_handshake_over_udp` helper) so it can tolerate those early
/// short-header 1-RTT packets — stashing them for replay — instead of failing
/// closed the way a handshake-only loop must when it sees a non-long-header
/// packet.
async fn accept(
cx: &Cx,
mut endpoint: QuicUdpEndpoint,
server_tls: &QuicServerTls,
config: &QuicConfig,
) -> Result<(QuicLink, Vec<ReceivedPacket>), QuicTransportError> {
cx.checkpoint().map_err(|_| QuicTransportError::Cancelled)?;
let mut driver = QuicHandshakeDriver::server(server_tls.config.clone(), ATP_QUIC_ALPN.to_vec())
.map_err(map_tls_error)?;
// RFC 9001 §5.2: Initial keys derive from the client's original DCID. Both
// peers agree on the protocol constant (see ATP_QUIC_INITIAL_DCID).
driver
.install_initial_keys(ATP_QUIC_INITIAL_DCID)
.map_err(map_tls_error)?;
let server_scid = ConnectionId::new(ATP_QUIC_SERVER_SCID)
.map_err(|err| QuicTransportError::Quic(format!("server scid: {err}")))?;
let mut server_pn = 0u64;
let mut peer: Option<(SocketAddr, ConnectionId)> = None;
let mut early_data: Vec<ReceivedPacket> = Vec::new();
let mut last_flight: Vec<OutgoingPacket> = Vec::new();
// `accept_timeout` bounds the WHOLE accept, and each receive waits at most
// one recovery PTO so lost flights are re-offered on a real cadence
// instead of only when a stale long-header packet happens to arrive.
let accept_started = Instant::now();
let mut flights = 0usize;
let mut last_early_data_resend: Option<Instant> = None;
while flights < HANDSHAKE_MAX_FLIGHTS {
if driver.is_complete() {
break;
}
let remaining = config
.accept_timeout
.saturating_sub(accept_started.elapsed());
if remaining.is_zero() {
return Err(QuicTransportError::Timeout {
operation: "quic server accept handshake",
timeout: config.accept_timeout,
});
}
let received = match crate::time::timeout(
cx.now(),
remaining.min(HANDSHAKE_RECOVERY_RESEND_PTO),
endpoint.receive_batch(cx, INBOUND_PUMP_BATCH),
)
.await
{
Ok(Ok(packets)) => packets,
Ok(Err(err)) => return Err(map_udp_error(err)),
Err(_elapsed) => {
// Quiet PTO window: if we have a peer and a flight, re-offer
// it — the client may be waiting on a lost server flight.
if peer.is_some() && !last_flight.is_empty() {
flights = flights.saturating_add(1);
endpoint
.send_batch(cx, &last_flight)
.await
.map_err(map_udp_error)?;
}
continue;
}
};
for packet in received {
if is_long_header(&packet.data) {
let client_cid = match driver.recv_handshake_packet(&packet.data) {
Ok(client_cid) => client_cid,
Err(err) if is_stale_handshake_packet_error(&err) => {
if !last_flight.is_empty() {
flights = flights.saturating_add(1);
endpoint
.send_batch(cx, &last_flight)
.await
.map_err(map_udp_error)?;
}
continue;
}
Err(err) => return Err(map_tls_error(err)),
};
if peer.is_none() {
peer = Some((packet.src_addr, client_cid));
}
if let Some((addr, dst_cid)) = peer {
let sent = send_server_handshake_flight(
cx,
&mut endpoint,
&mut driver,
addr,
dst_cid,
server_scid,
&mut server_pn,
)
.await?;
flights = flights.saturating_add(1);
if !sent.is_empty() {
last_flight = sent;
} else if !driver.is_complete() && !last_flight.is_empty() {
endpoint
.send_batch(cx, &last_flight)
.await
.map_err(map_udp_error)?;
}
}
} else {
// The client completed the handshake first and is already sending
// 1-RTT data. Stash it; it is replayed into the link below.
// While WE are still incomplete this is also hard evidence the
// client's final flight (its Finished) was lost — re-offer our
// flight (rate-limited) so the client's data plane sees a
// long-header packet and re-sends its Finished
// (br-asupersync-jmri58). Early-data packets deliberately do
// NOT consume the flight budget.
early_data.push(packet);
if !driver.is_complete()
&& !last_flight.is_empty()
&& last_early_data_resend
.is_none_or(|at| at.elapsed() >= HANDSHAKE_RECOVERY_RESEND_PTO)
{
last_early_data_resend = Some(Instant::now());
endpoint
.send_batch(cx, &last_flight)
.await
.map_err(map_udp_error)?;
}
}
}
}
if !driver.is_complete() {
return Err(QuicTransportError::Timeout {
operation: "quic server accept handshake",
timeout: config.accept_timeout,
});
}
let (peer_addr, _peer_cid) = peer
.ok_or_else(|| QuicTransportError::Quic("server handshake learned no peer".to_string()))?;
let link = link_from_handshake(cx, driver, endpoint, peer_addr, StreamRole::Server, config)?;
Ok((link, early_data))
}
// ─── Sender session ─────────────────────────────────────────────────────────
#[derive(Debug)]
struct NativeQuicAimdPacer {
controller: DatagramRateController,
controller_config: DatagramRateConfig,
last_decision: Option<DatagramRateDecision>,
shared_cap_active: bool,
sample_clock_micros: u64,
last_round_symbols_sent: u64,
last_round_pacing_rate_bps: u64,
last_round_send_wall: Duration,
last_round_rtt: Option<Duration>,
last_round_loss_fraction: f64,
last_pending_rank: Option<u64>,
last_pending_rank_deficit: Option<u64>,
}
impl Default for NativeQuicAimdPacer {
fn default() -> Self {
let controller_config = quic_datagram_rate_config(&QuicConfig::default());
Self {
controller: DatagramRateController::new(controller_config),
controller_config,
last_decision: None,
shared_cap_active: false,
sample_clock_micros: 0,
last_round_symbols_sent: 0,
last_round_pacing_rate_bps: 0,
last_round_send_wall: Duration::ZERO,
last_round_rtt: None,
last_round_loss_fraction: 0.0,
last_pending_rank: None,
last_pending_rank_deficit: None,
}
}
}
impl NativeQuicAimdPacer {
fn cap_bps(&self) -> Option<u64> {
self.shared_cap_active
.then(|| {
self.last_decision
.map(|decision| decision.pacing_bytes_per_s)
})
.flatten()
}
/// Sender-observed delivery loss from the last completed round. Used to seed
/// `pacing.path_loss_rate` (Bug A, MATRIX-126) so the round-0 clean-link ramp
/// disables after a high delivery loss instead of re-arming and flooding.
fn observed_loss(&self) -> f64 {
self.last_round_loss_fraction
}
fn shared_decision(&self) -> Option<DatagramRateDecision> {
self.last_decision
}
fn apply_shared_decision_to_pacing(
shared_decision: Option<DatagramRateDecision>,
pacing: &mut QuicSprayPacingDecision,
symbol_frame_len: usize,
config: &QuicConfig,
) {
let Some(decision) = shared_decision else {
return;
};
let symbol_bytes = u64::try_from(symbol_frame_len.max(1))
.unwrap_or(u64::MAX)
.max(1);
let fanout = config.datagram_fanout.max(1);
let shared_rate = decision.pacing_bytes_per_s.max(1);
pacing.pacing_rate_bps = pacing.pacing_rate_bps.min(shared_rate).max(1);
let inflight_symbols = usize::try_from(
decision
.inflight_limit_bytes
.checked_div(symbol_bytes)
.unwrap_or(0)
.max(1),
)
.unwrap_or(usize::MAX);
let budget_symbols = usize::try_from(
decision
.send_budget_bytes
.checked_div(symbol_bytes)
.unwrap_or(0)
.max(1),
)
.unwrap_or(usize::MAX);
pacing.cwnd_symbols = inflight_symbols;
pacing.cwnd_share_symbols = (inflight_symbols / fanout).max(1);
pacing.burst_cap_share_symbols = pacing.burst_cap_share_symbols.min(budget_symbols).max(1);
pacing.max_burst_symbols = pacing.max_burst_symbols.min(budget_symbols).max(1);
pacing.path_cwnd_bytes = decision.inflight_limit_bytes.max(symbol_bytes);
let sender_loss = f64::from(decision.sender_loss_fraction_ppm) / 1_000_000.0;
pacing.path_loss_rate = pacing.path_loss_rate.max(sender_loss);
pacing.congestion_loss_rate = pacing.congestion_loss_rate.max(sender_loss);
if sender_loss > 0.0 {
pacing.limiter = super::QuicSprayPacingLimiter::LossBackoff;
}
super::update_quic_pacing_pause(
pacing,
symbol_frame_len,
config.max_spray_symbols_per_flush,
);
pacing.max_burst_symbols = pacing.max_burst_symbols.min(budget_symbols).max(1);
}
fn sender_delivery_loss_for_repair(&self, receiver_loss: Option<f64>) -> Option<f64> {
if self.last_round_symbols_sent == 0 || !self.last_round_loss_fraction.is_finite() {
return None;
}
let receiver_loss = receiver_loss
.filter(|loss| loss.is_finite())
.unwrap_or(0.0)
.clamp(0.0, 0.90);
let sender_loss = self.last_round_loss_fraction.clamp(0.0, 0.90);
(sender_loss > receiver_loss + f64::EPSILON).then_some(sender_loss)
}
fn record_spray(&mut self, symbols_sent: u64, pacing_rate_bps: u64, send_wall: Duration) {
self.last_round_symbols_sent = symbols_sent;
self.last_round_pacing_rate_bps = pacing_rate_bps;
self.last_round_send_wall = send_wall;
self.last_round_rtt = None;
}
fn record_spray_with_pacing(
&mut self,
symbols_sent: u64,
pacing: &QuicSprayPacingDecision,
send_wall: Duration,
) {
self.record_spray(symbols_sent, pacing.pacing_rate_bps, send_wall);
self.last_round_rtt = duration_from_secs(pacing.path_rtt_s);
}
fn lossy_repair_feedback_enabled(config: &QuicConfig) -> bool {
super::quic_loss_target_pacing_cap_bps(config).is_some()
}
fn last_round_send_wall_s(&self) -> f64 {
self.last_round_send_wall.as_secs_f64().max(0.000_001)
}
fn round_payload_bps(&self, symbols: u64, config: &QuicConfig) -> f64 {
symbols.saturating_mul(u64::from(config.symbol_size.max(1))) as f64
/ self.last_round_send_wall_s()
}
fn receiver_delivery_bps(&self, need: &QuicNeedMore, config: &QuicConfig) -> Option<f64> {
need.round_symbols_observed
.or(need.round_symbols_accepted)
.map(|observed| {
self.round_payload_bps(observed.min(self.last_round_symbols_sent), config)
})
}
fn progress_delivery_bps(&mut self, config: &QuicConfig, need: &QuicNeedMore) -> Option<f64> {
if !Self::lossy_repair_feedback_enabled(config) {
self.last_pending_rank = need.pending_rank;
self.last_pending_rank_deficit = need.pending_rank_deficit;
return None;
}
let progress_accounted = need.pending_rank.is_some()
|| need.pending_rank_columns.is_some()
|| need.pending_rank_deficit.is_some()
|| need.pending_decode_jobs.is_some();
let rank_delta = need.pending_rank.map(|rank| {
let delta = self
.last_pending_rank
.map_or(rank, |previous| rank.saturating_sub(previous));
self.last_pending_rank = Some(rank);
delta
});
let deficit_delta = need.pending_rank_deficit.and_then(|deficit| {
let delta = self
.last_pending_rank_deficit
.map(|previous| previous.saturating_sub(deficit));
self.last_pending_rank_deficit = Some(deficit);
delta
});
let progress_symbols = rank_delta.unwrap_or(0).max(deficit_delta.unwrap_or(0));
if progress_symbols == 0 {
progress_accounted.then_some(0.0)
} else {
Some(self.round_payload_bps(progress_symbols, config))
}
}
fn progress_congestion_loss(
config: &QuicConfig,
progress_delivery_bps: f64,
offered_bps: f64,
) -> Option<f64> {
if !Self::lossy_repair_feedback_enabled(config)
|| !offered_bps.is_finite()
|| offered_bps <= 0.0
{
return None;
}
let delivery_ratio = (progress_delivery_bps / offered_bps).clamp(0.0, 1.0);
if delivery_ratio >= QUIC_LOSS_TARGET_PROGRESS_STALL_RATIO {
return None;
}
Some(
(1.0 - delivery_ratio)
.max(
super::quic_aimd_loss_decrease_threshold(config)
+ QUIC_LOSS_TARGET_PROGRESS_LOSS_MARGIN,
)
.clamp(0.0, 0.90),
)
}
fn ensure_controller_config(&mut self, config: &QuicConfig) {
let desired = quic_datagram_rate_config(config);
if desired != self.controller_config {
self.controller = DatagramRateController::new(desired);
self.controller_config = desired;
self.last_decision = None;
self.shared_cap_active = false;
self.sample_clock_micros = 0;
}
}
fn observe_shared_controller(
&mut self,
config: &QuicConfig,
need: &QuicNeedMore,
delivered_payload_bytes: u64,
loss: f64,
) -> DatagramRateDecision {
self.ensure_controller_config(config);
let symbol_bytes = u64::from(config.symbol_size.max(1));
let sent_payload_bytes = self
.last_round_symbols_sent
.saturating_mul(symbol_bytes.max(1));
let lost_bytes = ((sent_payload_bytes as f64) * loss.clamp(0.0, 1.0))
.ceil()
.clamp(0.0, sent_payload_bytes as f64) as u64;
let delivered_payload_bytes = delivered_payload_bytes.min(sent_payload_bytes);
let bytes_in_flight = sent_payload_bytes.saturating_sub(delivered_payload_bytes);
let receiver_credit_bytes = quic_need_more_receiver_credit_bytes(config, need);
let receiver_window_bytes =
receiver_credit_bytes.map(|credit| credit.saturating_add(bytes_in_flight));
let rtt_micros = self
.last_round_rtt
.map(duration_micros_u64)
.unwrap_or_else(|| duration_micros_u64(self.last_round_send_wall))
.max(1);
if self.sample_clock_micros == 0 {
self.sample_clock_micros = 1;
let _ = self.controller.observe(DatagramRateSample {
now_micros: self.sample_clock_micros,
sent_bytes: 1,
acked_bytes: 1,
lost_bytes: 0,
bytes_in_flight: 0,
rtt_micros: Some(rtt_micros),
receiver_credit_bytes: None,
receiver_window_bytes: None,
});
}
let send_wall_micros = duration_micros_u64(self.last_round_send_wall).max(1);
self.sample_clock_micros = self.sample_clock_micros.saturating_add(send_wall_micros);
let sample = DatagramRateSample {
now_micros: self.sample_clock_micros,
sent_bytes: sent_payload_bytes,
acked_bytes: delivered_payload_bytes,
lost_bytes,
bytes_in_flight,
rtt_micros: Some(rtt_micros),
receiver_credit_bytes,
receiver_window_bytes,
};
let decision = self.controller.observe(sample);
self.last_decision = Some(decision);
decision
}
fn observe_datagram_ack_sample(
&mut self,
cx: &Cx,
config: &QuicConfig,
sample: DatagramRateSample,
) -> DatagramRateDecision {
self.ensure_controller_config(config);
self.sample_clock_micros = self.sample_clock_micros.max(sample.now_micros);
let decision = self.controller.observe(sample);
self.last_decision = Some(decision);
self.last_round_loss_fraction = f64::from(decision.sender_loss_fraction_ppm) / 1_000_000.0;
self.shared_cap_active = decision.loss_limited
|| decision.flow_control_limited
|| self.last_round_loss_fraction > super::quic_aimd_loss_decrease_threshold(config);
let now_micros = sample.now_micros.to_string();
let sent_bytes = sample.sent_bytes.to_string();
let acked_bytes = sample.acked_bytes.to_string();
let lost_bytes = sample.lost_bytes.to_string();
let bytes_in_flight = sample.bytes_in_flight.to_string();
let rtt_micros = sample
.rtt_micros
.map_or_else(|| "none".to_string(), |rtt| rtt.to_string());
let pacing_bytes_per_s = decision.pacing_bytes_per_s.to_string();
let delivery_rate_bytes_per_s = decision.delivery_rate_bytes_per_s.to_string();
let sender_loss_fraction_ppm = decision.sender_loss_fraction_ppm.to_string();
let cwnd_bytes = decision.cwnd_bytes.to_string();
let inflight_limit_bytes = decision.inflight_limit_bytes.to_string();
let loss_limited = if decision.loss_limited {
"true"
} else {
"false"
};
cx.trace_with_fields(
"atp_quic.spray.ack_clock_feedback",
&[
("now_micros", now_micros.as_str()),
("sent_bytes", sent_bytes.as_str()),
("acked_bytes", acked_bytes.as_str()),
("lost_bytes", lost_bytes.as_str()),
("bytes_in_flight", bytes_in_flight.as_str()),
("rtt_micros", rtt_micros.as_str()),
("pacing_bytes_per_s", pacing_bytes_per_s.as_str()),
(
"delivery_rate_bytes_per_s",
delivery_rate_bytes_per_s.as_str(),
),
(
"sender_loss_fraction_ppm",
sender_loss_fraction_ppm.as_str(),
),
("loss_limited", loss_limited),
("cwnd_bytes", cwnd_bytes.as_str()),
// Correlation-safe field budget: <=12 explicit fields
// (br-asupersync-an0t8o). send_budget_bytes is derivable from
// inflight_limit_bytes and bytes_in_flight above.
("inflight_limit_bytes", inflight_limit_bytes.as_str()),
],
);
decision
}
fn observe_datagram_ack_samples(
&mut self,
cx: &Cx,
config: &QuicConfig,
samples: Vec<DatagramRateSample>,
) -> usize {
let mut observed = 0usize;
for sample in samples {
self.observe_datagram_ack_sample(cx, config, sample);
observed = observed.saturating_add(1);
}
observed
}
fn datagram_send_admission(
&mut self,
config: &QuicConfig,
now_micros: u64,
bytes_in_flight: u64,
bytes: u64,
) -> DatagramSendAdmission {
self.ensure_controller_config(config);
let now_micros = now_micros.max(1);
let bytes = bytes.max(1);
let admission = self
.controller
.admit_send(now_micros, bytes_in_flight, bytes, None, None);
if !admission.is_send() {
return admission;
}
if self
.controller
.try_send(now_micros, bytes_in_flight, bytes, None)
{
DatagramSendAdmission::Send
} else {
self.controller
.admit_send(now_micros, bytes_in_flight, bytes, None, None)
}
}
#[allow(clippy::cast_precision_loss, clippy::cast_possible_truncation)]
fn observe_need_more(&mut self, cx: &Cx, config: &QuicConfig, need: &QuicNeedMore) {
let sent = self.last_round_symbols_sent;
if sent == 0 {
return;
}
// Sender-side DELIVERY loss: symbols sent vs symbols the receiver OBSERVED on
// the wire. This is the only signal that sees QUEUE/WIRE DROPS. The receiver's
// `round_loss_fraction` counts loss only AMONG ARRIVED symbols, so during a
// ~98% queue overflow it reads ~0.0000 and AIMD never trips its backoff — the
// sender keeps flooding the overflowing queue until PTO timeout / non-convergence
// (MATRIX-123/124/125, bead asupersync-atp-dataplane-redesign-317hxr.2.5.1).
// Drive AIMD from delivery loss; keep the receiver fraction only as a floor (it
// can exceed delivery loss when arrived symbols are themselves corrupt/duplicate).
let delivery_loss = need
.round_symbols_observed
.or(need.round_symbols_accepted)
.map(|observed| {
let observed = observed.min(sent);
(1.0 - observed as f64 / sent as f64).clamp(0.0, 0.90)
});
let receiver_loss = need.round_loss_fraction.filter(|loss| loss.is_finite());
let wire_loss = match (delivery_loss, receiver_loss) {
(Some(delivered), Some(received)) => delivered.max(received),
(Some(delivered), None) => delivered,
(None, Some(received)) => received,
(None, None) => 0.0,
}
.clamp(0.0, 0.90);
let progress_delivery_bps = self.progress_delivery_bps(config, need);
let offered_bps = self.round_payload_bps(sent, config).max(1.0);
let progress_loss = progress_delivery_bps
.and_then(|delivery_bps| {
Self::progress_congestion_loss(config, delivery_bps, offered_bps)
})
.unwrap_or(0.0);
let loss = wire_loss.max(progress_loss).clamp(0.0, 0.90);
let receiver_delivery_bps = self.receiver_delivery_bps(need, config);
let observed_delivery_bps = receiver_delivery_bps.or(progress_delivery_bps);
let delivered_payload_bytes = observed_delivery_bps
.map(|bps| {
(bps * self.last_round_send_wall_s())
.ceil()
.clamp(0.0, u64::MAX as f64) as u64
})
.unwrap_or_else(|| {
need.round_symbols_observed
.or(need.round_symbols_accepted)
.unwrap_or(0)
.min(sent)
.saturating_mul(u64::from(config.symbol_size.max(1)))
});
let decision = self.observe_shared_controller(config, need, delivered_payload_bytes, loss);
self.last_round_loss_fraction =
(f64::from(decision.sender_loss_fraction_ppm) / 1_000_000.0).max(loss);
self.shared_cap_active =
self.last_round_loss_fraction > super::quic_aimd_loss_decrease_threshold(config);
let cap = self
.cap_bps()
.map_or_else(|| "none".to_string(), |cap| cap.to_string());
let sent = sent.to_string();
let observed = need
.round_symbols_observed
.or(need.round_symbols_accepted)
.unwrap_or(0)
.to_string();
let loss = format!("{:.4}", self.last_round_loss_fraction);
let progress_delivery = progress_delivery_bps
.map(|bps| format!("{bps:.0}"))
.unwrap_or_else(|| "none".to_string());
let offered = format!("{offered_bps:.0}");
let pending_rank = need.pending_rank.unwrap_or(0).to_string();
let pending_rank_columns = need.pending_rank_columns.unwrap_or(0).to_string();
let pending_rank_deficit = need.pending_rank_deficit.unwrap_or(0).to_string();
let pending_decode_jobs = need.pending_decode_jobs.unwrap_or(0).to_string();
let shared_pacing_bytes_per_s = decision.pacing_bytes_per_s.to_string();
let shared_cwnd_bytes = decision.cwnd_bytes.to_string();
cx.trace_with_fields(
"atp_quic.spray.aimd_feedback",
&[
("round_symbols_sent", sent.as_str()),
("round_symbols_observed", observed.as_str()),
("round_loss_fraction", loss.as_str()),
("progress_delivery_bps", progress_delivery.as_str()),
("offered_bps", offered.as_str()),
("pending_rank", pending_rank.as_str()),
("pending_rank_columns", pending_rank_columns.as_str()),
("pending_rank_deficit", pending_rank_deficit.as_str()),
("pending_decode_jobs", pending_decode_jobs.as_str()),
// Correlation-safe field budget: <=12 explicit fields
// (br-asupersync-an0t8o). The shared controller's full
// decision (delivery rate, loss ppm, inflight limit,
// loss_limited) is already traced per sample on
// atp_quic.spray.ack_clock_feedback; keep only the two
// round-level anchors here.
(
"shared_pacing_bytes_per_s",
shared_pacing_bytes_per_s.as_str(),
),
("shared_cwnd_bytes", shared_cwnd_bytes.as_str()),
("aimd_cap_bps", cap.as_str()),
],
);
}
}
fn refresh_datagram_ack_clocked_pacing(
cx: &Cx,
link: &mut QuicLink,
config: &QuicConfig,
aimd: &mut NativeQuicAimdPacer,
pacing: &mut QuicSprayPacingDecision,
trace_epoch: u64,
) -> usize {
let observed =
aimd.observe_datagram_ack_samples(cx, config, link.drain_datagram_rate_samples());
if observed == 0 {
return 0;
}
*pacing = link.spray_pacing_decision(
config,
aimd.cap_bps(),
aimd.shared_decision(),
aimd.observed_loss(),
);
link.update_spray_packet_coalescing(pacing);
link.data_plane_pacer
.configure_with_shared_decision(pacing, aimd.shared_decision());
pacing.trace_epoch(cx, trace_epoch);
observed
}
fn quic_datagram_rate_config(config: &QuicConfig) -> DatagramRateConfig {
let loss_target_cap = super::quic_loss_target_pacing_cap_bps(config);
let initial = loss_target_cap
.unwrap_or(super::QUIC_DEFAULT_COLD_START_PACING_BYTES_PER_S as u64)
.clamp(super::QUIC_AIMD_MIN_RATE_BPS, super::QUIC_AIMD_MAX_RATE_BPS);
let max_pacing = loss_target_cap
.unwrap_or(super::QUIC_AIMD_MAX_RATE_BPS)
.clamp(super::QUIC_AIMD_MIN_RATE_BPS, super::QUIC_AIMD_MAX_RATE_BPS);
let initial_cwnd = if loss_target_cap.is_some() {
initial
.saturating_mul(QUIC_LOSSY_COLD_START_RTT_MICROS)
.div_ceil(1_000_000)
.clamp(16 * 1024, 256 * 1024)
} else {
256 * 1024
};
DatagramRateConfig {
initial_pacing_bytes_per_s: initial,
min_pacing_bytes_per_s: super::QUIC_AIMD_MIN_RATE_BPS,
max_pacing_bytes_per_s: max_pacing,
initial_cwnd_bytes: initial_cwnd,
min_cwnd_bytes: 16 * 1024,
max_cwnd_bytes: 16 * 1024 * 1024,
pacing_gain: 1.0,
cwnd_gain: 1.0,
loss_backoff_threshold: super::quic_aimd_loss_decrease_threshold(config),
loss_backoff_factor: super::QUIC_AIMD_MULTIPLICATIVE_DECREASE,
loss_delivery_headroom: QUIC_LOSS_TARGET_DELIVERY_BACKOFF_HEADROOM,
receiver_window_gain: 1.0,
min_receiver_window_bytes: 16 * 1024,
max_receiver_window_bytes: 16 * 1024 * 1024,
min_rtt_window_micros: 10_000_000,
}
}
fn duration_from_secs(seconds: f64) -> Option<Duration> {
(seconds.is_finite() && seconds > 0.0).then(|| Duration::from_secs_f64(seconds))
}
fn duration_micros_u64(duration: Duration) -> u64 {
u64::try_from(duration.as_micros()).unwrap_or(u64::MAX)
}
fn quic_need_more_receiver_credit_bytes(config: &QuicConfig, need: &QuicNeedMore) -> Option<u64> {
let symbol_bytes = u64::from(config.symbol_size.max(1));
let requested = need_more_requested_symbol_count(need);
let deficit = need
.pending_rank_deficit
.or(need.repair_base_deficit_symbols)
.unwrap_or(0);
let credit_symbols = deficit
.saturating_mul(2)
.max(requested)
.max(if !need.pending.is_empty() { 32 } else { 0 });
(credit_symbols > 0).then_some(credit_symbols.saturating_mul(symbol_bytes))
}
/// Spray a round of symbols for the `pending` entries over a live link, pacing on
/// the bounded outbound DATAGRAM queue. Mirrors `super::spray_native_symbol_round`
/// but interleaves real UDP flushes so a large object never drops symbols before
/// they reach the wire. `with_source` selects the initial source+repair spray vs
/// a repair-only batch.
async fn spray_round(
cx: &Cx,
link: &mut QuicLink,
control: &mut NativeQuicFrameTransport,
manifest: &TransferManifest,
encoders: &mut [QuicEntryEncoder],
pending: &std::collections::BTreeSet<u32>,
round: u32,
config: &QuicConfig,
symbol_auth: Option<&SecurityContext>,
with_source: bool,
aimd: &mut NativeQuicAimdPacer,
) -> Result<u64, QuicTransportError> {
let tag = super::transfer_tag(&manifest.transfer_id);
let repair_batch = super::repair_batch_per_block(config);
let drop_one_in = config.debug_drop_one_in;
let mut pacing = link.spray_pacing_decision(
config,
aimd.cap_bps(),
aimd.shared_decision(),
aimd.observed_loss(),
);
refresh_datagram_ack_clocked_pacing(cx, link, config, aimd, &mut pacing, 1);
let repair_round = !with_source;
if repair_round {
enforce_native_repair_round_pacing(&mut pacing, link.symbol_datagram_frame_len);
}
let clean_ramp_max_pacing_bps = super::quic_round0_clean_ramp_max_pacing_bps(&pacing);
let mut clean_ramp =
super::quic_round0_clean_ramp_enabled(config, &pacing, with_source).then(|| {
super::QuicRound0CleanPacingRamp::new_with_burst_cap(
clean_ramp_max_pacing_bps,
config.max_spray_symbols_per_flush,
)
});
if clean_ramp.is_some() {
quic_rqtrace!(
"sender-native: round0_clean_pacing_ramp enabled start_rate_Bps={} step_bytes={} max_rate_Bps={} datagram_fanout={} datagram_frame_bytes={} burst_symbols={}",
pacing.pacing_rate_bps,
super::QUIC_ROUND0_CLEAN_RAMP_STEP_BYTES,
clean_ramp_max_pacing_bps,
config.datagram_fanout.max(1),
link.symbol_datagram_frame_len,
pacing.max_burst_symbols,
);
}
pacing.trace_epoch(cx, u64::from(!with_source));
link.reset_sender_handoff_trace();
let send_start = Instant::now();
let mut sent = 0u64;
let mut sprayed = 0u64;
let mut cursor_updates = Vec::new();
let mut stopped_for_feedback = false;
// Keep the native sender handoff continuous across source blocks; high-BDP
// and future fanout paths need paced windows, not per-block partial flushes.
let mut handoff_batch = Vec::with_capacity(link.spray_handoff_symbol_limit(&pacing));
'entries: for index in pending {
let Some(entry) = encoders.iter_mut().find(|entry| entry.index == *index) else {
continue;
};
if with_source && link.has_pending_fountain_feedback() {
stopped_for_feedback = true;
break;
}
cx.checkpoint().map_err(|_| QuicTransportError::Cancelled)?;
for block_idx in 0..entry.block_count(config)? {
if with_source && link.has_pending_fountain_feedback() {
stopped_for_feedback = true;
break 'entries;
}
let sbn = u8::try_from(block_idx).map_err(|_| {
QuicTransportError::Integrity("source block index exceeded u8 range".to_string())
})?;
let block = entry.read_block(cx, sbn, config).await?;
let already = entry.repair_cursor(sbn);
let target_repair = if with_source {
super::initial_repair_per_block(block.len(), config)
} else {
already.saturating_add(repair_batch)
};
let repair_count = target_repair.saturating_sub(already);
if !with_source && repair_count == 0 {
entry.set_repair_cursor(sbn, target_repair);
continue;
}
let mut pipeline = super::encoding_pipeline(config);
let object_id = entry.object_id;
let entry_index = entry.index;
let encoded = if with_source {
super::EitherNativeEncoding::Source(pipeline.encode_single_block_with_repair(
object_id,
sbn,
&block,
target_repair,
))
} else {
super::EitherNativeEncoding::Repair(pipeline.encode_single_block_repair_range(
object_id,
sbn,
&block,
already,
repair_count,
))
};
let mut generated_for_block = 0u64;
let mut queued_for_block = 0u64;
let mut emitted_repair_symbols = 0usize;
for encoded_symbol in encoded {
let symbol = encoded_symbol
.map_err(|err| QuicTransportError::Control(err.to_string()))?
.into_symbol();
generated_for_block = generated_for_block.saturating_add(1);
if symbol.kind().is_repair() {
emitted_repair_symbols = emitted_repair_symbols.saturating_add(1);
}
// Deterministic test-only symbol loss: skip on the initial spray only,
// so the receiver must drive a repair round, and never on a repair round
// (otherwise it could fail to converge). Control frames are unaffected.
sprayed = sprayed.saturating_add(1);
if with_source && drop_one_in > 0 && sprayed % u64::from(drop_one_in) == 0 {
continue;
}
queued_for_block = queued_for_block.saturating_add(1);
let auth_tag = symbol_auth.map(|ctx| *ctx.sign_symbol_tag(&symbol).as_bytes());
handoff_batch.push(NativeQuicSpraySymbol {
symbol,
entry: entry_index,
auth_tag,
});
let handoff_limit = link.spray_handoff_symbol_limit(&pacing);
if handoff_batch.len() >= handoff_limit {
link.spray_symbol_batch(
cx,
control,
tag,
&handoff_batch,
config,
&mut pacing,
aimd,
repair_round,
)
.await?;
if refresh_datagram_ack_clocked_pacing(cx, link, config, aimd, &mut pacing, 4)
> 0
&& !super::quic_round0_clean_ramp_enabled(config, &pacing, with_source)
{
let _ = clean_ramp.take();
}
if repair_round {
enforce_native_repair_round_pacing(
&mut pacing,
link.symbol_datagram_frame_len,
);
}
let batch_len = u64::try_from(handoff_batch.len()).unwrap_or(u64::MAX);
sent = sent.saturating_add(batch_len);
for _ in 0..handoff_batch.len() {
if let Some(ramp) = &mut clean_ramp {
if let Some(report) =
ramp.observe_datagram(&mut pacing, link.symbol_datagram_frame_len)
{
quic_rqtrace!(
"sender-native: round0_clean_rate_ramp sent_datagrams={} sent_bytes={} old_rate_Bps={} new_rate_Bps={} next_step_bytes={} max_rate_Bps={}",
report.sent_datagrams,
report.sent_bytes,
report.old_rate_bps,
report.new_rate_bps,
report.next_step_bytes,
report.max_rate_bps,
);
}
}
}
if refresh_datagram_ack_clocked_pacing(cx, link, config, aimd, &mut pacing, 4)
> 0
&& aimd.observed_loss() >= QUIC_CLEAN_SPRAY_MAX_LOSS_RATE
{
let _ = clean_ramp.take();
}
if repair_round {
enforce_native_repair_round_pacing(
&mut pacing,
link.symbol_datagram_frame_len,
);
}
handoff_batch.clear();
handoff_batch.reserve(link.spray_handoff_symbol_limit(&pacing));
if with_source && link.has_pending_fountain_feedback() {
stopped_for_feedback = true;
break;
}
}
}
let mode = if with_source {
"initial"
} else {
"fallback_repair"
};
super::quic_progress(format_args!(
"sender: block_spray round={round} transfer={} entry={entry_index} sbn={sbn} mode={mode} generated_symbols={generated_for_block} queued_symbols={queued_for_block} repair_symbols={emitted_repair_symbols} repair_cursor_before={already} repair_cursor_after={target_repair} pacing_rate_bps={}",
manifest.transfer_id, pacing.pacing_rate_bps
));
if stopped_for_feedback {
let repair_cursor =
already.saturating_add(emitted_repair_symbols.min(repair_count));
cursor_updates.push((entry_index, sbn, repair_cursor));
trace_quic_initial_spray_cut_for_feedback(
cx,
sent,
link.pending_fountain_feedback_count(),
link.conn.pending_outbound_datagram_count(),
link.conn.transport().bytes_in_flight(),
link.conn.transport().congestion_window_bytes(),
entry_index,
sbn,
repair_cursor,
);
break 'entries;
}
cursor_updates.push((entry_index, sbn, target_repair));
}
}
if !stopped_for_feedback && !handoff_batch.is_empty() {
link.spray_symbol_batch(
cx,
control,
tag,
&handoff_batch,
config,
&mut pacing,
aimd,
repair_round,
)
.await?;
if refresh_datagram_ack_clocked_pacing(cx, link, config, aimd, &mut pacing, 4) > 0
&& !super::quic_round0_clean_ramp_enabled(config, &pacing, with_source)
{
let _ = clean_ramp.take();
}
if repair_round {
enforce_native_repair_round_pacing(&mut pacing, link.symbol_datagram_frame_len);
}
let batch_len = u64::try_from(handoff_batch.len()).unwrap_or(u64::MAX);
sent = sent.saturating_add(batch_len);
for _ in 0..handoff_batch.len() {
if let Some(ramp) = &mut clean_ramp {
if let Some(report) =
ramp.observe_datagram(&mut pacing, link.symbol_datagram_frame_len)
{
quic_rqtrace!(
"sender-native: round0_clean_rate_ramp sent_datagrams={} sent_bytes={} old_rate_Bps={} new_rate_Bps={} next_step_bytes={} max_rate_Bps={}",
report.sent_datagrams,
report.sent_bytes,
report.old_rate_bps,
report.new_rate_bps,
report.next_step_bytes,
report.max_rate_bps,
);
}
}
}
if refresh_datagram_ack_clocked_pacing(cx, link, config, aimd, &mut pacing, 4) > 0
&& aimd.observed_loss() >= QUIC_CLEAN_SPRAY_MAX_LOSS_RATE
{
let _ = clean_ramp.take();
}
if repair_round {
enforce_native_repair_round_pacing(&mut pacing, link.symbol_datagram_frame_len);
}
}
for (entry_index, sbn, target_repair) in cursor_updates {
if let Some(entry) = encoders.iter_mut().find(|entry| entry.index == entry_index) {
entry.set_repair_cursor(sbn, target_repair);
}
}
aimd.record_spray_with_pacing(sent, &pacing, send_start.elapsed());
link.finish_paced_spray_round(cx, control, sent, &pacing)
.await?;
Ok(sent)
}
/// Send the specific systematic source symbols a receiver requested, paced.
async fn spray_source_requests(
cx: &Cx,
link: &mut QuicLink,
control: &mut NativeQuicFrameTransport,
manifest: &TransferManifest,
encoders: &[QuicEntryEncoder],
requests: &[QuicSourceSymbolRequest],
round: u32,
config: &QuicConfig,
symbol_auth: Option<&SecurityContext>,
aimd: &mut NativeQuicAimdPacer,
) -> Result<u64, QuicTransportError> {
let tag = super::transfer_tag(&manifest.transfer_id);
let mut pacing = link.spray_pacing_decision(
config,
aimd.cap_bps(),
aimd.shared_decision(),
aimd.observed_loss(),
);
refresh_datagram_ack_clocked_pacing(cx, link, config, aimd, &mut pacing, 2);
pacing.trace_epoch(cx, 2);
link.reset_sender_handoff_trace();
let send_start = Instant::now();
let mut sent = 0u64;
let mut handoff_batch = Vec::with_capacity(link.spray_handoff_symbol_limit(&pacing));
for request in requests {
cx.checkpoint().map_err(|_| QuicTransportError::Cancelled)?;
let enc = encoders
.iter()
.find(|entry| entry.index == request.entry)
.ok_or_else(|| {
QuicTransportError::Integrity(format!(
"receiver requested source symbol for unknown entry {}",
request.entry
))
})?;
let block = enc.read_block(cx, request.sbn, config).await?;
let symbol_size = usize::from(config.symbol_size.max(1));
let block_k = block.len().div_ceil(symbol_size).max(1);
let esi = usize::try_from(request.esi).map_err(|_| {
QuicTransportError::Integrity("source request ESI does not fit usize".to_string())
})?;
if esi >= block_k {
return Err(QuicTransportError::Integrity(format!(
"source request esi {} outside entry {} block {} K={block_k}",
request.esi, enc.index, request.sbn
)));
}
let start = esi * symbol_size;
let end = (start + symbol_size).min(block.len());
let mut buffer = vec![0u8; symbol_size];
if start < end {
buffer[..end - start].copy_from_slice(&block[start..end]);
}
let symbol = Symbol::new(
crate::types::symbol::SymbolId::new(enc.object_id, request.sbn, request.esi),
buffer,
crate::types::symbol::SymbolKind::Source,
);
let auth_tag = symbol_auth.map(|ctx| *ctx.sign_symbol_tag(&symbol).as_bytes());
handoff_batch.push(NativeQuicSpraySymbol {
symbol,
entry: request.entry,
auth_tag,
});
super::quic_progress(format_args!(
"sender: source_symbol round={round} transfer={} entry={} sbn={} esi={} queued_symbols=1 pacing_rate_bps={}",
manifest.transfer_id, request.entry, request.sbn, request.esi, pacing.pacing_rate_bps
));
if handoff_batch.len() >= link.spray_handoff_symbol_limit(&pacing) {
link.spray_symbol_batch(
cx,
control,
tag,
&handoff_batch,
config,
&mut pacing,
aimd,
false,
)
.await?;
refresh_datagram_ack_clocked_pacing(cx, link, config, aimd, &mut pacing, 2);
sent = sent.saturating_add(u64::try_from(handoff_batch.len()).unwrap_or(u64::MAX));
handoff_batch.clear();
handoff_batch.reserve(link.spray_handoff_symbol_limit(&pacing));
}
}
if !handoff_batch.is_empty() {
link.spray_symbol_batch(
cx,
control,
tag,
&handoff_batch,
config,
&mut pacing,
aimd,
false,
)
.await?;
refresh_datagram_ack_clocked_pacing(cx, link, config, aimd, &mut pacing, 2);
sent = sent.saturating_add(u64::try_from(handoff_batch.len()).unwrap_or(u64::MAX));
}
aimd.record_spray_with_pacing(sent, &pacing, send_start.elapsed());
link.finish_paced_spray_round(cx, control, sent, &pacing)
.await?;
Ok(sent)
}
/// Send fresh repair symbols for the specific source blocks a receiver still lacks.
async fn spray_block_repair_requests(
cx: &Cx,
link: &mut QuicLink,
control: &mut NativeQuicFrameTransport,
manifest: &TransferManifest,
encoders: &mut [QuicEntryEncoder],
requests: &[QuicBlockRepairRequest],
round: u32,
config: &QuicConfig,
symbol_auth: Option<&SecurityContext>,
aimd: &mut NativeQuicAimdPacer,
) -> Result<u64, QuicTransportError> {
let tag = super::transfer_tag(&manifest.transfer_id);
let mut pacing = link.spray_pacing_decision(
config,
aimd.cap_bps(),
aimd.shared_decision(),
aimd.observed_loss(),
);
refresh_datagram_ack_clocked_pacing(cx, link, config, aimd, &mut pacing, 3);
enforce_native_repair_round_pacing(&mut pacing, link.symbol_datagram_frame_len);
pacing.trace_epoch(cx, 3);
link.reset_sender_handoff_trace();
let send_start = Instant::now();
let mut sent = 0u64;
let mut cursor_updates = Vec::with_capacity(requests.len());
let mut handoff_batch = Vec::with_capacity(link.spray_handoff_symbol_limit(&pacing));
for request in requests {
cx.checkpoint().map_err(|_| QuicTransportError::Cancelled)?;
let Some(enc_index) = encoders
.iter()
.position(|entry| entry.index == request.entry)
else {
return Err(QuicTransportError::Integrity(format!(
"receiver requested repair block for unknown entry {}",
request.entry
)));
};
let repair_count = usize::try_from(request.symbols).map_err(|_| {
QuicTransportError::Integrity("repair symbol count does not fit usize".to_string())
})?;
let (block, already, target_repair, object_id, entry_index) = {
let enc = &mut encoders[enc_index];
let block = enc.read_block(cx, request.sbn, config).await?;
let already = enc.repair_cursor(request.sbn);
(
block,
already,
already.saturating_add(repair_count),
enc.object_id,
enc.index,
)
};
if entry_index != request.entry {
return Err(QuicTransportError::Integrity(format!(
"receiver requested repair block for unknown entry {}",
request.entry
)));
}
let mut pipeline = super::encoding_pipeline(config);
let mut emitted_for_request = 0u64;
for encoded in pipeline.encode_single_block_repair_range(
object_id,
request.sbn,
&block,
already,
repair_count,
) {
let symbol = encoded
.map_err(|err| QuicTransportError::Control(err.to_string()))?
.into_symbol();
let auth_tag = symbol_auth.map(|ctx| *ctx.sign_symbol_tag(&symbol).as_bytes());
handoff_batch.push(NativeQuicSpraySymbol {
symbol,
entry: entry_index,
auth_tag,
});
emitted_for_request = emitted_for_request.saturating_add(1);
if handoff_batch.len() >= link.spray_handoff_symbol_limit(&pacing) {
link.spray_symbol_batch(
cx,
control,
tag,
&handoff_batch,
config,
&mut pacing,
aimd,
true,
)
.await?;
refresh_datagram_ack_clocked_pacing(cx, link, config, aimd, &mut pacing, 3);
enforce_native_repair_round_pacing(&mut pacing, link.symbol_datagram_frame_len);
sent = sent.saturating_add(u64::try_from(handoff_batch.len()).unwrap_or(u64::MAX));
handoff_batch.clear();
handoff_batch.reserve(link.spray_handoff_symbol_limit(&pacing));
}
}
if emitted_for_request != u64::from(request.symbols) {
return Err(QuicTransportError::Integrity(format!(
"sender emitted {emitted_for_request} repair symbols for receiver-requested deficit {} on entry {} block {}",
request.symbols, entry_index, request.sbn
)));
}
cursor_updates.push((enc_index, request.sbn, target_repair));
super::quic_progress(format_args!(
"sender: repair_block round={round} transfer={} entry={} sbn={} requested_symbols={} emitted_symbols={} repair_cursor_before={} repair_cursor_after={} pacing_rate_bps={}",
manifest.transfer_id,
entry_index,
request.sbn,
request.symbols,
emitted_for_request,
already,
target_repair,
pacing.pacing_rate_bps
));
quic_rqtrace!(
"sender: repair_block entry={} sbn={} requested_symbols={} emitted_symbols={} repair_cursor_before={} repair_cursor_after={} pacing_rate_bps={}",
entry_index,
request.sbn,
request.symbols,
emitted_for_request,
already,
target_repair,
pacing.pacing_rate_bps,
);
}
if !handoff_batch.is_empty() {
link.spray_symbol_batch(
cx,
control,
tag,
&handoff_batch,
config,
&mut pacing,
aimd,
true,
)
.await?;
refresh_datagram_ack_clocked_pacing(cx, link, config, aimd, &mut pacing, 3);
enforce_native_repair_round_pacing(&mut pacing, link.symbol_datagram_frame_len);
sent = sent.saturating_add(u64::try_from(handoff_batch.len()).unwrap_or(u64::MAX));
}
for (enc_index, sbn, target_repair) in cursor_updates {
let Some(enc) = encoders.get_mut(enc_index) else {
return Err(QuicTransportError::Integrity(
"repair cursor update referenced missing encoder".to_string(),
));
};
enc.set_repair_cursor(sbn, target_repair);
}
aimd.record_spray_with_pacing(sent, &pacing, send_start.elapsed());
link.finish_paced_spray_round(cx, control, sent, &pacing)
.await?;
Ok(sent)
}
/// Receiver `HandshakeAck` parse (mirrors `super::receive_native_sender_hello_ack`).
fn parse_hello_ack(frame: &Frame) -> Result<QuicHelloAck, QuicTransportError> {
if frame.frame_type() != FrameType::HandshakeAck {
return Err(QuicTransportError::HandshakeRejected(format!(
"unexpected {:?} frame while awaiting HandshakeAck",
frame.frame_type()
)));
}
let ack: QuicHelloAck = super::parse_json(frame).map_err(|err| {
QuicTransportError::HandshakeRejected(format!("invalid handshake acknowledgement: {err}"))
})?;
if !ack.accepted {
return Err(QuicTransportError::HandshakeRejected(
ack.reason
.clone()
.unwrap_or_else(|| "no reason given".to_string()),
));
}
Ok(ack)
}
async fn drive_native_source_stream_flush(
cx: &Cx,
link: &mut QuicLink,
idle_timeout: Duration,
drain_all: bool,
) -> Result<(), QuicTransportError> {
let started = Instant::now();
let mut last_progress = Instant::now();
let mut made_progress = false;
let mut recent_stream_frames = Vec::new();
let mut stall_slices = QuicStallSliceAccounting::default();
let mut pending_stall: Option<QuicSendStallReason> = None;
loop {
cx.checkpoint().map_err(|_| QuicTransportError::Cancelled)?;
// Limiter report: attribute the previous iteration's wait to the gate
// it classified; an iteration that made progress takes its slice
// without attributing it.
let held = stall_slices.take_slice(started.elapsed());
if let Some(reason) = pending_stall.take() {
link.note_limiter_stall(reason, held);
}
let pending_frames = link.conn.pending_stream_frame_count();
if pending_frames == 0 {
return Ok(());
}
let pending_bytes = link.conn.pending_stream_data_bytes();
let flushed = link.flush(cx).await?;
made_progress |= flushed > 0;
if flushed > 0 {
last_progress = Instant::now();
let frames = link.last_flushed_stream_frames();
if !frames.is_empty() {
recent_stream_frames = frames;
}
}
quic_rqtrace!(
"sender: native_source_stream_flush pending_frames={} pending_bytes={} flushed={} drain_all={} in_flight={} cwnd={}",
pending_frames,
pending_bytes,
flushed,
drain_all,
link.conn.transport().bytes_in_flight(),
link.conn.transport().congestion_window_bytes(),
);
if link.conn.pending_stream_frame_count() == 0 {
return Ok(());
}
// Only a BLOCKED flush waits for inbound (ACKs unblock it); a healthy
// mid-stream cycle drains inbound opportunistically with a zero
// timeout. Waiting a grace per 512 KiB quantum convoyed the sender
// and receiver into RTT-scale lockstep (~47 MB/s at any pacing rate).
if flushed == 0 {
let paced_credit = link
.paced_source_stream
.map(|stream| link.conn.stream_send_credit_remaining(stream));
pending_stall = Some(classify_blocked_flush(
link.conn
.transport()
.can_send(QUIC_DATA_PLANE_TELEMETRY_PACKET_BYTES),
paced_credit,
));
}
let pump_timeout = if flushed == 0 {
INBOUND_PUMP_DRAIN_GRACE
} else {
Duration::ZERO
};
let pumped = link.pump_inbound_for(cx, pump_timeout).await?;
if pumped > 0 {
last_progress = Instant::now();
let latest_stream_ack_ranges = link.latest_stream_ack_ranges.clone();
let retransmit_frames =
link.drain_ack_gap_lost_stream_frames_for_retransmit(&latest_stream_ack_ranges);
if !retransmit_frames.is_empty() {
let retransmitted = link
.retransmit_stream_frames(
cx,
&retransmit_frames,
"source_stream_ack_gap_retransmit",
)
.await?;
if retransmitted > 0 {
last_progress = Instant::now();
recent_stream_frames = link.last_flushed_stream_frames();
continue;
}
}
}
if link.conn.pending_stream_frame_count() == 0 {
return Ok(());
}
if !drain_all && made_progress {
return Ok(());
}
if flushed == 0 && pumped == 0 && last_progress.elapsed() >= link.app_loss_stall_pto {
// Declare PTO loss BEFORE the pending-frames guard (see
// `next_control_frame_with_source_stream_recovery`): pending
// frames behind a cwnd-blocked flush must not starve the loss
// timeout, or the in-flight accounting never releases and the
// flush livelocks until the idle timeout.
let _ = link.expire_app_data_loss_timeout(cx, "flush QUIC source stream")?;
if link.conn.has_pending_stream_frames() {
continue;
}
let mut retransmit_frames = link.drain_limited_in_flight_stream_frames_for_retransmit(
QUIC_SOURCE_STREAM_PTO_RETRANSMIT_MAX_PACKETS,
);
if retransmit_frames.is_empty() {
retransmit_frames = recent_stream_frames.clone();
}
if !retransmit_frames.is_empty() {
let retransmitted = link
.retransmit_stream_frames(cx, &retransmit_frames, "source_stream_pto")
.await?;
if retransmitted > 0 {
last_progress = Instant::now();
recent_stream_frames = link.last_flushed_stream_frames();
continue;
}
}
last_progress = Instant::now();
}
if started.elapsed() >= idle_timeout {
return Err(QuicTransportError::Timeout {
operation: "flush QUIC source stream",
timeout: idle_timeout,
});
}
}
}
/// Wait until the paced source stream may admit new payload bytes: the
/// send queue is flushed below its cap, sent-but-unacked bytes are back under
/// the runaway guard, and (when the receiver negotiated a bounded window) at
/// least `min_credit` bytes of send credit are available. All waits are
/// ACK/pacer clocked and run the same gap/PTO recovery as the flush loop so a
/// lossy window cannot wedge the gate; a credit-blocked sender with nothing
/// left in flight keep-alive-pings so the receiver's ACKs (which re-attach
/// MAX_STREAM_DATA advertisements) keep flowing.
async fn wait_source_stream_send_admission(
cx: &Cx,
link: &mut QuicLink,
stream: StreamId,
min_credit: u64,
config: &QuicConfig,
) -> Result<(), QuicTransportError> {
let gate_started = Instant::now();
let mut last_progress = Instant::now();
let mut progress_marker = (u64::MAX, u64::MAX);
let mut stall_slices = QuicStallSliceAccounting::default();
let mut pending_stall: Option<QuicSendStallReason> = None;
loop {
// Limiter report: the previous iteration's wait belongs to the gate
// it classified below (queue cap, stream credit or admission cap).
let gate_elapsed = gate_started.elapsed();
let held = stall_slices.take_slice(gate_elapsed);
if let Some(reason) = pending_stall.take() {
link.note_limiter_stall(reason, held);
}
if gate_elapsed >= config.idle_timeout {
return Err(QuicTransportError::Timeout {
operation: "source stream send admission",
timeout: config.idle_timeout,
});
}
if link.conn.pending_stream_data_bytes() > QUIC_SOURCE_STREAM_SEND_QUEUE_MAX_BYTES {
pending_stall = classify_source_stream_admission(true, true, true);
drive_native_source_stream_flush(cx, link, config.idle_timeout, false).await?;
continue;
}
let credit_remaining = link.conn.stream_send_credit_remaining(stream);
link.limiter.min_stream_send_credit_bytes = Some(
link.limiter
.min_stream_send_credit_bytes
.map_or(credit_remaining, |min| min.min(credit_remaining)),
);
let credit_ok = min_credit == 0 || credit_remaining >= min_credit;
let admission_cap = link.source_stream_unacked_admission_max();
let unacked_ok = link.stream_unacked_bytes <= admission_cap;
let bottleneck_bytes_per_s = link
.stream_rate_controller
.as_ref()
.map_or(0, SourceStreamRatePacer::bottleneck_bytes_per_s);
link.limiter.observe_source_stream(
admission_cap,
link.stream_unacked_bytes,
credit_remaining,
bottleneck_bytes_per_s,
);
if credit_ok && unacked_ok {
return Ok(());
}
pending_stall = classify_source_stream_admission(false, credit_ok, unacked_ok);
// Sender-gated receive-window growth (A4): while the peer's credit is
// the only gate holding us, tell it so with STREAM_DATA_BLOCKED —
// once per window of credit, and only on a transfer that has never
// repaired a loss (see `source_stream_window_request_due`). A
// queue-limited or lossy path therefore never asks and keeps the
// 2 MiB path law. The frame rides the gate's existing flush below;
// asking must never fail the transfer or add a wait of its own.
if !credit_ok && unacked_ok {
let send_limit = link.conn.stream_send_limit(stream).unwrap_or(0);
let window = link.source_stream_send_window.unwrap_or(0);
if super::source_stream_window_request_due(
&mut link.source_stream_window_request,
send_limit,
window,
link.limiter.retransmit_batches,
) && link.conn.report_stream_data_blocked(cx, stream).is_ok()
{
link.limiter.stream_window_requests =
link.limiter.stream_window_requests.saturating_add(1);
}
}
// Any observable movement counts as progress. A stricter
// ≥64KB-per-PTO watermark was measured WORSE (gate36: tree_small/good
// 8.3→22.2s uniform): steady drainage kept re-basing the watermark,
// the PTO drain fired every 200 ms, and its multi-MB re-flushes
// recreated the retransmit churn this gate exists to prevent.
let marker = (credit_remaining, link.stream_unacked_bytes);
if marker != progress_marker {
progress_marker = marker;
last_progress = Instant::now();
}
cx.checkpoint().map_err(|_| QuicTransportError::Cancelled)?;
let pumped = link.pump_inbound_for(cx, SOURCE_STREAM_PTO).await?;
// Give never-yet-sent pending frames (e.g. a tree manifest's tail on
// the control stream) a shot at whatever cwnd the latest ACKs freed:
// the gate's non-stall path otherwise never flushes, so pending
// control frames starved behind the retransmit-priority queue for the
// whole wait (br-asupersync-daqxbz).
if link.conn.has_pending_stream_frames() || link.conn.has_pending_control_frames() {
let _ = link.flush(cx).await?;
}
// The PTO drain must key on lack of PROGRESS, not on a silent pump:
// the gate's own keep-alive pings (and the receiver's ACKs of them)
// keep `pumped > 0` forever, and a tail loss at the credit-window
// edge has no later packets to trip the 3-packet gap detector — that
// combination degenerated recovery to ~1 packet per PTO (measured:
// tree_small/good trickling at 8 KB per 252 ms for seconds).
let stalled = last_progress.elapsed() >= link.app_loss_stall_pto;
if pumped > 0 && !stalled {
let ranges = link.latest_stream_ack_ranges.clone();
let frames = link.drain_ack_gap_lost_stream_frames_for_retransmit(&ranges);
if !frames.is_empty() {
link.retransmit_stream_frames(cx, &frames, "source_stream_unacked_gate_retransmit")
.await?;
}
} else {
let _ = link.expire_app_data_loss_timeout(cx, "source stream unacked gate")?;
let frames = link.drain_limited_in_flight_stream_frames_for_retransmit(
QUIC_SOURCE_STREAM_PTO_RETRANSMIT_MAX_PACKETS,
);
if !frames.is_empty() {
link.retransmit_stream_frames(cx, &frames, "source_stream_unacked_gate_pto")
.await?;
last_progress = Instant::now();
} else if !credit_ok {
link.conn.queue_ping(cx)?;
link.flush(cx).await?;
last_progress = Instant::now();
}
}
}
}
async fn send_native_source_stream_entries_pumped(
cx: &Cx,
link: &mut QuicLink,
stream: StreamId,
prepared: &QuicPreparedSource,
config: &QuicConfig,
) -> Result<u64, QuicTransportError> {
let max_chunk = link
.max_app_payload
.saturating_sub(usize::try_from(QUIC_STREAM_PACKET_OVERHEAD_BUDGET).unwrap_or(usize::MAX))
.max(1);
let flush_chunk = usize::try_from(QUIC_SOURCE_STREAM_FLUSH_BYTES).unwrap_or(usize::MAX);
let mut buf = vec![0_u8; max_chunk.min(flush_chunk).max(1)];
let mut streamed = 0u64;
let mut queued_since_flush = 0u64;
for entry in &prepared.entries {
cx.checkpoint().map_err(|_| QuicTransportError::Cancelled)?;
if entry.size == 0 {
continue;
}
let mut file = crate::fs::File::open(&entry.abs_path)
.await
.map_err(|err| {
QuicTransportError::Source(format!("{}: {err}", entry.abs_path.display()))
})?;
let mut hasher = Sha256::new();
let mut read = 0u64;
loop {
cx.checkpoint().map_err(|_| QuicTransportError::Cancelled)?;
let n = file.read(&mut buf).await.map_err(|err| {
QuicTransportError::Source(format!("{}: {err}", entry.abs_path.display()))
})?;
if n == 0 {
break;
}
read = read.saturating_add(u64::try_from(n).unwrap_or(u64::MAX));
if read > entry.size {
return Err(QuicTransportError::Source(format!(
"{} grew while streaming QUIC source bytes (read {read} bytes, manifest size {})",
entry.abs_path.display(),
entry.size
)));
}
hasher.update(&buf[..n]);
let n_u64 = u64::try_from(n).unwrap_or(u64::MAX);
// Never write past the receiver's bounded window: `write_stream_bytes`
// consumes send credit at queue time and fails on exhaustion, so a
// credit shortfall waits on the ACK/advertisement clock instead.
if link.conn.stream_send_credit_remaining(stream) < n_u64 {
drive_native_source_stream_flush(cx, link, config.idle_timeout, false).await?;
queued_since_flush = 0;
wait_source_stream_send_admission(cx, link, stream, n_u64, config).await?;
}
link.conn
.write_stream_bytes(cx, stream, Bytes::copy_from_slice(&buf[..n]), false)?;
streamed = streamed.saturating_add(n_u64);
queued_since_flush = queued_since_flush.saturating_add(n_u64);
if queued_since_flush >= QUIC_SOURCE_STREAM_FLUSH_BYTES {
drive_native_source_stream_flush(cx, link, config.idle_timeout, false).await?;
queued_since_flush = 0;
// Bound sender-side buffering (RSS), sent-but-unacked bytes
// (runaway guard), and the next flush window's send credit:
// without the first, the disk reader queues the entire file
// ahead of the pacer (~540 MB RSS on a 500 MB transfer);
// without the second, a mis-estimated pacing rate streams the
// file into a loss gap faster than the receiver can
// reassemble; the third keeps writes inside the receiver's
// advertised reassembly window. All waits are ACK/pacer
// clocked, not spins.
// Credit is gated per-write above (demanding a full flush
// window of credit here quantized the transfer into one
// window per RTT); this wait only enforces the queue and
// unacked ceilings.
wait_source_stream_send_admission(cx, link, stream, 0, config).await?;
}
}
if read != entry.size {
return Err(QuicTransportError::Source(format!(
"{} changed while streaming QUIC source bytes (read {read} bytes, manifest size {})",
entry.abs_path.display(),
entry.size
)));
}
let got_sha = hex_encode(&hasher.finalize());
if got_sha != entry.sha256_hex {
return Err(QuicTransportError::Integrity(format!(
"{} changed while streaming QUIC source bytes (sha256 {got_sha}, manifest {})",
entry.abs_path.display(),
entry.sha256_hex
)));
}
}
// Source exhausted: everything from here (tail drain, FIN, proof
// exchange) is app-limited for the delivery sampler.
link.stream_source_exhausted = true;
link.conn
.write_stream_bytes(cx, stream, Bytes::new(), true)?;
drive_native_source_stream_flush(cx, link, config.idle_timeout, true).await?;
Ok(streamed)
}
#[allow(
clippy::cast_possible_truncation,
clippy::cast_precision_loss,
clippy::cast_sign_loss
)]
#[allow(dead_code)]
fn source_stream_repair_tail_requests(
manifest: &TransferManifest,
config: &QuicConfig,
) -> Result<Vec<QuicBlockRepairRequest>, QuicTransportError> {
let mut requests = Vec::new();
for entry in &manifest.entries {
for block_idx in 0..super::block_count_for_len(entry.size, config)? {
let sbn = u8::try_from(block_idx).map_err(|_| QuicTransportError::TooLarge {
size: entry.size,
max: u64::try_from(config.max_block_size.max(1))
.unwrap_or(u64::MAX)
.saturating_mul(u64::from(u8::MAX) + 1),
})?;
let block_start = u64::from(sbn)
.saturating_mul(u64::try_from(config.max_block_size.max(1)).unwrap_or(u64::MAX));
let block_len = usize::try_from(
entry
.size
.saturating_sub(block_start)
.min(u64::try_from(config.max_block_size.max(1)).unwrap_or(u64::MAX)),
)
.unwrap_or(usize::MAX);
let block_source_symbols = block_len
.min(config.max_block_size.max(1))
.div_ceil(usize::from(config.symbol_size.max(1)))
.max(1);
let configured_overhead = (config.repair_overhead - 1.0).max(0.0);
let loss_floor =
config.round0_loss_target.max(0.0) * QUIC_SOURCE_STREAM_REPAIR_LOSS_MULTIPLIER;
let repair_fraction = configured_overhead.max(loss_floor);
let repair = ((block_source_symbols as f64) * repair_fraction).ceil() as usize;
if repair == 0 {
continue;
}
requests.push(QuicBlockRepairRequest {
entry: entry.index,
sbn,
symbols: u32::try_from(repair).unwrap_or(u32::MAX),
});
}
}
Ok(requests)
}
/// Drive a full ATP-over-QUIC send over an established link: Hello, manifest,
/// initial symbol spray, then the fountain feedback loop until Proof or the
/// feedback-round budget is exhausted.
async fn run_sender_session(
cx: &Cx,
link: &mut QuicLink,
prepared: &QuicPreparedSource,
config: &QuicConfig,
peer_id: &str,
) -> Result<SendReport, QuicTransportError> {
let config = prepared.effective_config(config);
let config = &config;
config.validate()?;
super::validate_quic_manifest(&prepared.manifest, config)?;
let manifest = &prepared.manifest;
let symbol_auth = config.symbol_auth_context()?;
let symbol_auth_enabled = symbol_auth.is_some();
let delta_sender_nonce = if manifest.delta_manifest.is_some()
&& super::quic_delta_control_auth_context(config).is_some()
{
Some(super::fresh_quic_delta_nonce(cx, b"sender", None)?)
} else {
None
};
let mut control = NativeQuicFrameTransport::open(cx, &mut link.conn)?;
let offered_source_stream =
if super::quic_native_source_stream_enabled(manifest.total_bytes, config, &link.conn) {
Some(link.conn.open_local_bidi(cx)?)
} else {
None
};
super::send_native_sender_hello(
cx,
&mut link.conn,
&mut control,
config,
peer_id,
symbol_auth_enabled,
offered_source_stream,
manifest.total_bytes,
delta_sender_nonce,
)?;
link.flush(cx).await?;
let hello_frames = link.last_flushed_stream_frames();
let ack_frame = link
.next_control_frame_with_stream_pto(
cx,
&mut control,
"receive sender handshake ack",
&hello_frames,
"sender_hello_pto",
)
.await?;
let ack = parse_hello_ack(&ack_frame)?;
let delta_handshake = super::validate_quic_delta_ack(delta_sender_nonce, &ack)?;
let source_stream = match (offered_source_stream, ack.source_stream) {
(Some(stream), true) => Some(stream),
(None, true) => {
return Err(QuicTransportError::HandshakeRejected(
"receiver accepted an unoffered QUIC source stream".to_string(),
));
}
_ => None,
};
if let (Some(stream), Some(window)) = (source_stream, ack.source_stream_recv_window) {
// Honor the receiver's bounded reassembly window: cap initial send
// credit at one window; MAX_STREAM_DATA advertisements (re-attached to
// every receiver ACK) grow it as the receiver drains. This bounds
// sent-but-un-read bytes to ~one window, so a mis-seeded pacing rate
// can no longer pile a multi-MB backlog into the path (the 20 s PTO
// retransmit spirals) and receiver reassembly RSS stays flat.
link.conn.set_fresh_stream_send_limit(cx, stream, window)?;
link.source_stream_send_window = Some(window);
}
let delta_session = delta_handshake
.map(|handshake| {
super::derive_quic_delta_session(handshake, peer_id, &ack.peer_id, manifest)
})
.transpose()?;
if let Some(session) = delta_session {
let context = super::quic_delta_control_auth_context(config).ok_or_else(|| {
QuicTransportError::Config(
"QUIC delta control requires a strict authentication context".to_string(),
)
})?;
let envelope = super::make_quic_delta_manifest_envelope(context, session, manifest)?;
let frame = super::json_frame(FrameType::ObjectManifest, &envelope)?;
control.send(cx, &mut link.conn, &frame)?;
} else {
let mut full_manifest;
let manifest = if manifest.delta_manifest.is_some() {
full_manifest = manifest.clone();
full_manifest.delta_manifest = None;
&full_manifest
} else {
manifest
};
super::send_native_manifest(cx, &mut link.conn, &mut control, manifest)?;
}
link.flush(cx).await?;
let manifest_frames = link.last_flushed_stream_frames();
let mut accepted_delta_full_request = None;
if let Some(session) = delta_session {
let request_frame = loop {
let frame = link
.next_control_frame_with_stream_pto(
cx,
&mut control,
"receive bound QUIC delta request",
&manifest_frames,
"delta_manifest_pto",
)
.await?;
match frame.frame_type() {
FrameType::ObjectRequest => break frame,
FrameType::KeepAlive => {}
got => {
return Err(QuicTransportError::Unexpected {
got,
expected: "ObjectRequest | KeepAlive",
});
}
}
};
let request = super::parse_json::<super::QuicDeltaObjectRequest>(&request_frame)?;
let request_mode = super::validate_quic_delta_request(&request, session, manifest)?;
// Emit the transport ACK before source preparation or revalidation can
// delay the receiver's request-delivery gate.
link.flush(cx).await?;
match request_mode {
super::DeltaWireMode::FullObject => accepted_delta_full_request = Some(request),
super::DeltaWireMode::AlreadyInSync => {
super::validate_quic_prepared_delta_source_unchanged(cx, prepared, config).await?;
send_native_object_complete_for_round(cx, &mut link.conn, &mut control, 0, 0)?;
link.flush(cx).await?;
let completion_frames = link.last_flushed_stream_frames();
let proof_frame = loop {
let frame = link
.next_control_frame_with_stream_pto(
cx,
&mut control,
"receive bound QUIC delta proof",
&completion_frames,
"delta_object_complete_pto",
)
.await?;
match frame.frame_type() {
FrameType::Proof => break frame,
FrameType::KeepAlive => {}
got => {
return Err(QuicTransportError::Unexpected {
got,
expected: "Proof | KeepAlive",
});
}
}
};
let proof = super::parse_json::<super::QuicDeltaProof>(&proof_frame)?;
let receipt = super::validate_quic_delta_proof(proof, session, manifest)?;
super::send_native_close(cx, &mut link.conn, &mut control)?;
link.flush(cx).await?;
return Ok(SendReport {
transfer_id: manifest.transfer_id.clone(),
bytes_sent: 0,
files: super::manifest_logical_files(manifest),
symbols_sent: 0,
feedback_rounds: 0,
merkle_root_hex: manifest.merkle_root_hex.clone(),
receipt,
peer: link.peer,
});
}
super::DeltaWireMode::DeltaChunks => unreachable!(
"validate_quic_delta_request rejects missing-chunk mode in this rollout"
),
}
}
if let Some(source_stream) = source_stream {
let previous_paced_source_stream = link.paced_source_stream.replace(source_stream);
let source_pacing = link.source_stream_pacing_decision(config);
source_pacing.trace_epoch(cx, 0);
link.data_plane_pacer
.configure_source_stream(&source_pacing);
link.begin_source_stream_rate_control(source_pacing.pacing_rate_bps);
quic_rqtrace!(
"sender: native_source_stream_pacing rate_bps={} burst_symbols={} burst_bytes={} frame_bytes={}",
source_pacing.pacing_rate_bps,
source_pacing.max_burst_symbols,
link.data_plane_pacer.byte_pacer_burst_bytes,
source_stream_max_frame_bytes(),
);
let source_result: Result<SendReport, QuicTransportError> = erase_send(async {
let bytes_streamed =
send_native_source_stream_entries_pumped(cx, link, source_stream, prepared, config)
.await?;
if bytes_streamed != manifest.total_bytes {
return Err(QuicTransportError::Integrity(format!(
"source stream sent {bytes_streamed} bytes, expected {}",
manifest.total_bytes
)));
}
quic_rqtrace!(
"sender: native_source_stream sent bytes={} stream={}",
bytes_streamed,
source_stream.0
);
// Phase-cost breakdown for the pure-stream path (generate /
// protect / udp-send micros accumulate in record_flush on every
// flush; the spray paths already emit this — br-asupersync-oh6gm2
// flush-efficiency work needs it for stream bulk too).
link.trace_sender_handoff_summary(cx, "source_stream", bytes_streamed);
// Keep the source stream marked as ATP-paced through proof wait:
// PTO retransmits of lost source STREAM frames must not fall back
// under NewReno cwnd after the initial source send returns.
loop {
cx.checkpoint().map_err(|_| QuicTransportError::Cancelled)?;
let reply_frame = link
.next_control_frame_with_source_stream_recovery(
cx,
&mut control,
"receive stream-source proof",
)
.await?;
match reply_frame.frame_type() {
FrameType::Proof => {
let receipt = super::parse_json::<ReceiveReceipt>(&reply_frame)?;
super::send_native_close(cx, &mut link.conn, &mut control)?;
link.flush(cx).await?;
if !receipt.committed {
return Err(QuicTransportError::Integrity(
receipt
.reason
.clone()
.unwrap_or_else(|| "receiver did not commit".to_string()),
));
}
return Ok(SendReport {
transfer_id: manifest.transfer_id.clone(),
bytes_sent: manifest.total_bytes,
files: super::manifest_logical_files(manifest),
symbols_sent: 0,
feedback_rounds: 0,
merkle_root_hex: manifest.merkle_root_hex.clone(),
receipt,
peer: link.peer,
});
}
FrameType::KeepAlive => {}
FrameType::ObjectRequest => {
let need = super::parse_json::<QuicNeedMore>(&reply_frame)?;
quic_rqtrace!(
"sender: native_source_stream unexpected NeedMore round={} pending={} repair_blocks={} source_requests={} round_symbols_observed={} round_symbols_accepted={} round_loss_fraction={:.4}",
need.feedback_round,
need.pending.len(),
need.repair_blocks.len(),
need.source_symbols.len(),
need.round_symbols_observed.unwrap_or(0),
need.round_symbols_accepted.unwrap_or(0),
need.round_loss_fraction.unwrap_or(0.0),
);
return Err(QuicTransportError::NoConvergence {
rounds: need.feedback_round,
pending: need.pending.len(),
});
}
got => {
return Err(QuicTransportError::Unexpected {
got,
expected: "Proof | ObjectRequest | KeepAlive",
});
}
}
}
})
.await;
link.end_source_stream_rate_control();
link.paced_source_stream = previous_paced_source_stream;
return source_result;
}
let mut encoders = super::encoders_from_prepared_source(cx, prepared, config).await?;
let pending_all: std::collections::BTreeSet<u32> =
encoders.iter().map(|entry| entry.index).collect();
let mut aimd = NativeQuicAimdPacer::default();
let mut symbols_sent = erase_send(spray_round(
cx,
link,
&mut control,
manifest,
&mut encoders,
&pending_all,
0,
config,
symbol_auth.as_ref(),
true,
&mut aimd,
))
.await?;
// `spray_round` drains queued DATAGRAMs before returning so ObjectComplete
// cannot overtake unsent symbols in the connection's outbound queues.
link.trace_sender_handoff_summary(cx, "initial_source", symbols_sent);
send_native_object_complete_for_round(cx, &mut link.conn, &mut control, 0, symbols_sent)?;
link.flush(cx).await?;
let mut completion_frames = link.last_flushed_stream_frames();
super::quic_progress(format_args!(
"sender: object_complete_sent round=0 transfer={} round_symbols_sent={} stream_frames={}",
manifest.transfer_id,
symbols_sent,
completion_frames.len()
));
let mut feedback_rounds = 0u32;
loop {
cx.checkpoint().map_err(|_| QuicTransportError::Cancelled)?;
let reply_frame = link
.next_control_frame_with_stream_pto(
cx,
&mut control,
"receive proof or fountain feedback",
&completion_frames,
"object_complete_pto",
)
.await?;
let reply = match reply_frame.frame_type() {
FrameType::Proof => {
QuicControlReply::Proof(super::parse_json::<ReceiveReceipt>(&reply_frame)?)
}
FrameType::ObjectRequest => {
if let Some(expected) = accepted_delta_full_request.as_ref()
&& let Ok(duplicate) =
super::parse_json::<super::QuicDeltaObjectRequest>(&reply_frame)
{
if &duplicate != expected {
return Err(QuicTransportError::Integrity(
"receiver changed its bound QUIC delta request".to_string(),
));
}
continue;
}
QuicControlReply::NeedMore(super::parse_json::<QuicNeedMore>(&reply_frame)?)
}
FrameType::KeepAlive => continue,
got => {
return Err(QuicTransportError::Unexpected {
got,
expected: "Proof | ObjectRequest | KeepAlive",
});
}
};
match reply {
QuicControlReply::Proof(receipt) => {
super::quic_progress(format_args!(
"sender: proof_received transfer={} committed={} sha_ok={} merkle_ok={} feedback_rounds={} symbols_sent={}",
manifest.transfer_id,
receipt.committed,
receipt.sha_ok,
receipt.merkle_ok,
feedback_rounds,
symbols_sent
));
super::send_native_close(cx, &mut link.conn, &mut control)?;
link.flush(cx).await?;
if !receipt.committed {
return Err(QuicTransportError::Integrity(
receipt
.reason
.clone()
.unwrap_or_else(|| "receiver did not commit".to_string()),
));
}
return Ok(SendReport {
transfer_id: manifest.transfer_id.clone(),
bytes_sent: manifest.total_bytes,
files: super::manifest_logical_files(manifest),
symbols_sent,
feedback_rounds,
merkle_root_hex: manifest.merkle_root_hex.clone(),
receipt,
peer: link.peer,
});
}
QuicControlReply::NeedMore(need) => {
let (next_feedback_round, response_feedback_round) =
feedback_round_for_need_or_no_convergence(
feedback_rounds,
config.max_feedback_rounds,
need.feedback_round,
need.pending.len(),
)?;
aimd.observe_need_more(cx, config, &need);
feedback_rounds = next_feedback_round;
let requested_repair_symbols = need_more_repair_symbol_count(&need);
let base_deficit_symbols =
need_more_base_deficit_symbols(&need, requested_repair_symbols);
let sender_delivery_loss_for_repair =
aimd.sender_delivery_loss_for_repair(need.round_loss_fraction);
let loss_compensated_target_symbols =
loss_compensated_repair_target(base_deficit_symbols, need.round_loss_fraction);
let loss_compensated_target_symbols =
need_more_loss_compensated_target_symbols(&need, base_deficit_symbols)
.max(loss_compensated_target_symbols);
let repair_blocks_to_send = if need.repair_blocks.is_empty() {
Vec::new()
} else {
need.repair_blocks.clone()
};
let repair_symbols_to_emit = repair_block_symbol_count(&repair_blocks_to_send);
let loss_compensated_target_symbols =
loss_compensated_target_symbols.max(repair_symbols_to_emit);
let request_gap_to_target = need
.repair_request_gap_to_target_symbols
.unwrap_or_else(|| {
loss_compensated_target_symbols.saturating_sub(requested_repair_symbols)
});
let repair_detail = repair_block_trace_summary(&need.repair_blocks);
let sender_delivery_loss_for_repair_trace =
sender_delivery_loss_for_repair.unwrap_or(0.0);
super::quic_progress(format_args!(
"sender: need_more_received round={feedback_rounds} transfer={} pending={} repair_blocks={} requested_repair_symbols={} repair_symbols_to_emit={} source_requests={} round_symbols_observed={} round_symbols_accepted={} round_loss_fraction={:.4} repair_blocks_detail={}",
manifest.transfer_id,
need.pending.len(),
need.repair_blocks.len(),
requested_repair_symbols,
repair_symbols_to_emit,
need.source_symbols.len(),
need.round_symbols_observed.unwrap_or(0),
need.round_symbols_accepted.unwrap_or(0),
need.round_loss_fraction.unwrap_or(0.0),
repair_detail
));
quic_rqtrace!(
"sender: NeedMore round={feedback_rounds} pending={} repair_blocks={} base_deficit_symbols={} requested_repair_symbols={} repair_symbols_to_emit={} loss_compensated_target_symbols={} request_gap_to_target={} source_requests={} round_symbols_observed={} round_symbols_accepted={} round_loss_fraction={:.4} sender_delivery_loss_for_repair={:.4} max_feedback_rounds={} round_cap_exceeded={} repair_symbol_round_cap={} prior_total_symbols_sent={} prior_round_symbols_sent={} prior_pacing_rate_bps={} repair_blocks_detail={}",
need.pending.len(),
need.repair_blocks.len(),
base_deficit_symbols,
requested_repair_symbols,
repair_symbols_to_emit,
loss_compensated_target_symbols,
request_gap_to_target,
need.source_symbols.len(),
need.round_symbols_observed.unwrap_or(0),
need.round_symbols_accepted.unwrap_or(0),
need.round_loss_fraction.unwrap_or(0.0),
sender_delivery_loss_for_repair_trace,
config.max_feedback_rounds,
feedback_rounds > config.max_feedback_rounds,
need.repair_symbol_round_cap
.unwrap_or(super::MAX_REPAIR_SYMBOLS_PER_FEEDBACK_ROUND as u64),
symbols_sent,
aimd.last_round_symbols_sent,
aimd.last_round_pacing_rate_bps,
repair_detail,
);
trace_repair_block_deficits("sender", feedback_rounds, &need.repair_blocks);
super::trace_quic_sender_need_more(
cx,
feedback_rounds,
symbols_sent,
aimd.last_round_symbols_sent,
&need,
config,
None,
aimd.cap_bps(),
);
if need.pending.is_empty()
&& need.repair_blocks.is_empty()
&& need.source_symbols.is_empty()
{
super::trace_quic_sender_repair_round(
cx,
feedback_rounds,
super::quic_need_more_response_mode(&need),
symbols_sent,
0,
&need,
);
send_native_object_complete_for_round(
cx,
&mut link.conn,
&mut control,
response_feedback_round,
0,
)?;
link.flush(cx).await?;
completion_frames = link.last_flushed_stream_frames();
super::quic_progress(format_args!(
"sender: object_complete_sent round={} transfer={} round_symbols_sent=0 stream_frames={}",
response_feedback_round,
manifest.transfer_id,
completion_frames.len()
));
continue;
}
super::validate_need_more_feedback(manifest, config, &need)?;
let symbols_before = symbols_sent;
let response_mode = super::quic_need_more_response_mode(&need);
let sent = if !need.repair_blocks.is_empty() {
erase_send(spray_block_repair_requests(
cx,
link,
&mut control,
manifest,
&mut encoders,
&repair_blocks_to_send,
feedback_rounds,
config,
symbol_auth.as_ref(),
&mut aimd,
))
.await?
} else if need.source_symbols.is_empty() {
let fallback_pending = need.pending.iter().copied().collect();
erase_send(spray_round(
cx,
link,
&mut control,
manifest,
&mut encoders,
&fallback_pending,
feedback_rounds,
config,
symbol_auth.as_ref(),
false,
&mut aimd,
))
.await?
} else {
erase_send(spray_source_requests(
cx,
link,
&mut control,
manifest,
&encoders,
&need.source_symbols,
feedback_rounds,
config,
symbol_auth.as_ref(),
&mut aimd,
))
.await?
};
if !need.repair_blocks.is_empty() && sent != repair_symbols_to_emit {
return Err(QuicTransportError::Integrity(format!(
"sender emitted {sent} repair symbols for loss-compensated repair target {repair_symbols_to_emit}"
)));
}
symbols_sent = symbols_sent.saturating_add(sent);
trace_native_repair_accounting(
cx,
"sender",
feedback_rounds,
base_deficit_symbols,
requested_repair_symbols,
loss_compensated_target_symbols,
Some(sent),
&need,
);
super::trace_quic_sender_repair_round(
cx,
feedback_rounds,
response_mode,
symbols_before,
sent,
&need,
);
let emission_gap_to_target = loss_compensated_target_symbols.saturating_sub(sent);
quic_rqtrace!(
"sender: repair_round round={feedback_rounds} emitted_symbols={sent} requested_repair_symbols={requested_repair_symbols} loss_compensated_target_symbols={loss_compensated_target_symbols} emission_gap_to_target={emission_gap_to_target} total_symbols_sent={symbols_sent} pacing_rate_bps={} max_feedback_rounds={} round_cap_enforced=false",
aimd.last_round_pacing_rate_bps,
config.max_feedback_rounds,
);
// Flush this round's repair/source symbols before ObjectComplete
// (same ordering guarantee as the initial spray).
link.flush(cx).await?;
let handoff_phase = if need.repair_blocks.is_empty() {
"source_requests"
} else {
"repair_blocks"
};
link.trace_sender_handoff_summary(cx, handoff_phase, sent);
send_native_object_complete_for_round(
cx,
&mut link.conn,
&mut control,
response_feedback_round,
sent,
)?;
link.flush(cx).await?;
completion_frames = link.last_flushed_stream_frames();
super::quic_progress(format_args!(
"sender: object_complete_sent round={} transfer={} round_symbols_sent={} stream_frames={}",
response_feedback_round,
manifest.transfer_id,
sent,
completion_frames.len()
));
if !need.repair_blocks.is_empty() {
let _ = link
.drop_duplicate_need_more_resends(cx, &mut control, &need)
.await?;
}
}
}
}
}
// ─── Receiver session ───────────────────────────────────────────────────────
/// Incremental hash state maintained while staged writes remain strictly
/// sequential, so commit can skip the post-stream re-read + SHA-256 pass.
struct QuicInlineEntryHash {
sha: Sha256,
cid: crate::atp::object::ContentIdHasher,
hashed: u64,
}
impl QuicInlineEntryHash {
fn new() -> Self {
Self {
sha: Sha256::new(),
cid: crate::atp::object::ContentId::streaming(),
hashed: 0,
}
}
}
struct QuicStagedEntryReceive {
staging_path: PathBuf,
created: bool,
staging_file: Option<crate::fs::File>,
staging_cursor: Option<u64>,
staging_unflushed_bytes: usize,
cache_staging_file: bool,
sparse_files: bool,
/// `Some` while every accepted write has been sequential from offset 0;
/// dropped on the first out-of-order write (e.g. decoded FEC blocks
/// landing out of order), which falls back to the post-stream hash pass.
inline_hash: Option<QuicInlineEntryHash>,
}
impl QuicStagedEntryReceive {
#[cfg(test)]
fn new(staging_path: PathBuf, entry_size: u64, manifest_entries: usize) -> Self {
Self::new_with_options(
staging_path,
entry_size,
manifest_entries,
&QuicReceiveOptions::default(),
)
}
fn new_with_options(
staging_path: PathBuf,
entry_size: u64,
manifest_entries: usize,
options: &QuicReceiveOptions,
) -> Self {
Self {
staging_path,
created: false,
staging_file: None,
staging_cursor: None,
staging_unflushed_bytes: 0,
cache_staging_file: should_cache_quic_staging_file(entry_size, manifest_entries),
sparse_files: options.sparse_files(),
// Sparse reconstruction is verified from the staged file itself so
// seek/write mistakes cannot be hidden by a hash of inbound bytes.
inline_hash: (!options.sparse_files()).then(QuicInlineEntryHash::new),
}
}
/// Consume the inline hash if it covered the entire entry, yielding the
/// same `(size, content_id, content_sha256)` triple as
/// `hash_file_streaming` over the staged file (identical byte stream:
/// every accepted write was folded in sequentially from offset 0).
fn take_inline_hash_if_complete(
&mut self,
entry_size: u64,
) -> Option<(u64, crate::atp::object::ObjectId, [u8; 32])> {
let complete = self
.inline_hash
.as_ref()
.is_some_and(|state| state.hashed == entry_size);
if !complete {
return None;
}
let state = self.inline_hash.take()?;
let content_sha256: [u8; 32] = state.sha.finalize().into();
let content_id = crate::atp::object::ObjectId::content(state.cid.finalize());
Some((state.hashed, content_id, content_sha256))
}
async fn open_staging_file(
&mut self,
entry: &super::ManifestEntry,
) -> Result<crate::fs::File, QuicTransportError> {
if let Some(parent) = self.staging_path.parent() {
crate::fs::create_dir_all(parent).await?;
}
if self.created {
return Ok(crate::fs::OpenOptions::new()
.read(true)
.write(true)
.open(&self.staging_path)
.await?);
}
let file = crate::fs::File::create_new(&self.staging_path)
.await
.map_err(|err| {
if err.kind() == std::io::ErrorKind::AlreadyExists {
QuicTransportError::Integrity(format!(
"staging file already exists for entry {}",
entry.index
))
} else {
QuicTransportError::from(err)
}
})?;
file.set_len(entry.size).await?;
self.created = true;
Ok(file)
}
async fn write_block(
&mut self,
entry: &super::ManifestEntry,
block_sbn: u8,
data: &[u8],
config: &QuicConfig,
) -> Result<(), QuicTransportError> {
let offset = u64::from(block_sbn)
.checked_mul(config.max_block_size as u64)
.ok_or_else(|| {
QuicTransportError::Integrity("decoded block offset overflow".to_string())
})?;
if offset >= entry.size && !data.is_empty() {
return Err(QuicTransportError::Integrity(format!(
"decoded block {block_sbn} starts outside entry {} ({} bytes)",
entry.index, entry.size
)));
}
let end = offset.saturating_add(u64::try_from(data.len()).unwrap_or(u64::MAX));
if end > entry.size {
return Err(QuicTransportError::Integrity(format!(
"decoded block {block_sbn} overruns entry {}: end {end}, size {}",
entry.index, entry.size
)));
}
self.write_range(entry, offset, data).await
}
async fn write_range(
&mut self,
entry: &super::ManifestEntry,
offset: u64,
data: &[u8],
) -> Result<(), QuicTransportError> {
self.write_range_with_cursor_audit(entry, offset, data, quic_staging_cursor_audit_enabled())
.await
}
async fn write_range_with_cursor_audit(
&mut self,
entry: &super::ManifestEntry,
offset: u64,
data: &[u8],
audit_cursor: bool,
) -> Result<(), QuicTransportError> {
if offset > entry.size || (offset == entry.size && !data.is_empty()) {
return Err(QuicTransportError::Integrity(format!(
"staged write starts outside entry {}: offset {offset}, size {}",
entry.index, entry.size
)));
}
let end = offset.saturating_add(u64::try_from(data.len()).unwrap_or(u64::MAX));
if end > entry.size {
return Err(QuicTransportError::Integrity(format!(
"staged write overruns entry {}: end {end}, size {}",
entry.index, entry.size
)));
}
if let Some(state) = self.inline_hash.as_mut() {
if offset == state.hashed {
state.sha.update(data);
state.cid.update(data);
state.hashed = state.hashed.saturating_add(data.len() as u64);
} else {
// Out-of-order write: the inline hash no longer mirrors the
// file's byte order, so commit falls back to the streaming
// hash pass over the staged file.
self.inline_hash = None;
}
}
if self.cache_staging_file {
if self.staging_file.is_none() {
let file = self.open_staging_file(entry).await?;
self.staging_file = Some(file);
self.staging_cursor = None;
self.staging_unflushed_bytes = 0;
}
let next_cursor = offset
.checked_add(u64::try_from(data.len()).unwrap_or(u64::MAX))
.ok_or_else(|| {
QuicTransportError::Integrity(format!(
"entry {} staging cursor overflow",
entry.index
))
})?;
let unflushed_bytes = self.staging_unflushed_bytes.saturating_add(data.len());
let should_flush = unflushed_bytes >= QUIC_STAGE_BUFFER_BYTES;
let expected_cursor = self.staging_cursor;
{
let file = self.staging_file.as_mut().ok_or_else(|| {
QuicTransportError::Integrity(format!(
"entry {} staging file missing after open",
entry.index
))
})?;
if expected_cursor != Some(offset) {
file.seek(std::io::SeekFrom::Start(offset)).await?;
} else if audit_cursor {
let actual = file.stream_position().await?;
if actual != offset {
quic_rqtrace!(
"receiver: entry {} STAGING_CURSOR_DESYNC expected={} actual={} len={}",
entry.index,
offset,
actual,
data.len()
);
file.seek(std::io::SeekFrom::Start(offset)).await?;
}
}
if self.sparse_files {
crate::net::atp::transport_common::metadata::write_sparse_zero_runs(file, data)
.await?;
} else {
file.write_all(data).await?;
}
if should_flush {
file.flush().await?;
}
}
self.staging_cursor = Some(next_cursor);
self.staging_unflushed_bytes = if should_flush { 0 } else { unflushed_bytes };
return Ok(());
}
let mut file = self.open_staging_file(entry).await?;
file.seek(std::io::SeekFrom::Start(offset)).await?;
if self.sparse_files {
crate::net::atp::transport_common::metadata::write_sparse_zero_runs(&mut file, data)
.await?;
} else {
file.write_all(data).await?;
}
file.flush().await?;
Ok(())
}
async fn close_cached_staging_file(&mut self) -> Result<(), QuicTransportError> {
if let Some(mut file) = self.staging_file.take() {
file.flush().await?;
}
self.staging_cursor = None;
self.staging_unflushed_bytes = 0;
Ok(())
}
async fn flush_cached_staging_file(&mut self) -> Result<(), QuicTransportError> {
if let Some(file) = self.staging_file.as_mut() {
file.flush().await?;
}
self.staging_unflushed_bytes = 0;
Ok(())
}
async fn ensure_created(
&mut self,
entry: &super::ManifestEntry,
) -> Result<(), QuicTransportError> {
if self.created {
return Ok(());
}
let _ = self.open_staging_file(entry).await?;
Ok(())
}
}
fn should_cache_quic_staging_file(entry_size: u64, manifest_entries: usize) -> bool {
entry_size >= QUIC_STAGING_FILE_CACHE_MIN_BYTES
&& manifest_entries <= QUIC_STAGING_FILE_CACHE_MAX_ENTRIES
}
async fn flush_cached_quic_staging_files(
staged: &mut [QuicStagedEntryReceive],
) -> Result<(), QuicTransportError> {
for entry in staged {
entry.flush_cached_staging_file().await?;
}
Ok(())
}
fn quic_staging_nonce_hex() -> Result<String, QuicTransportError> {
let mut nonce = [0u8; 8];
getrandom::fill(&mut nonce)
.map_err(|err| QuicTransportError::Quic(format!("generate staging nonce: {err}")))?;
Ok(hex_encode(&nonce))
}
#[derive(Debug, Default, Clone, Copy)]
struct NativeReceiverIntakeStats {
drain_calls: u64,
symbols_observed: u64,
symbols_accepted: u64,
blocks_completed: u64,
drain_micros: u64,
pump_calls: u64,
pump_packets: u64,
pump_micros: u64,
staging_write_count: u64,
staging_write_bytes: u64,
staging_write_micros: u64,
}
impl NativeReceiverIntakeStats {
fn record_symbol_drain(
&mut self,
elapsed: Duration,
observed: u64,
accepted: u64,
completed_blocks: usize,
) {
self.drain_calls = self.drain_calls.saturating_add(1);
self.symbols_observed = self.symbols_observed.saturating_add(observed);
self.symbols_accepted = self.symbols_accepted.saturating_add(accepted);
self.blocks_completed = self
.blocks_completed
.saturating_add(u64::try_from(completed_blocks).unwrap_or(u64::MAX));
self.drain_micros = self
.drain_micros
.saturating_add(u64::try_from(elapsed.as_micros()).unwrap_or(u64::MAX));
}
fn record_pump(&mut self, elapsed: Duration, packets: usize) {
self.pump_calls = self.pump_calls.saturating_add(1);
self.pump_packets = self
.pump_packets
.saturating_add(u64::try_from(packets).unwrap_or(u64::MAX));
self.pump_micros = self
.pump_micros
.saturating_add(u64::try_from(elapsed.as_micros()).unwrap_or(u64::MAX));
}
fn record_staging_write(&mut self, elapsed: Duration, bytes: usize) {
self.staging_write_count = self.staging_write_count.saturating_add(1);
self.staging_write_bytes = self
.staging_write_bytes
.saturating_add(u64::try_from(bytes).unwrap_or(u64::MAX));
self.staging_write_micros = self
.staging_write_micros
.saturating_add(u64::try_from(elapsed.as_micros()).unwrap_or(u64::MAX));
}
fn record_completed_blocks(&mut self, completed_blocks: usize) {
self.blocks_completed = self
.blocks_completed
.saturating_add(u64::try_from(completed_blocks).unwrap_or(u64::MAX));
}
fn trace_need_more(&self, cx: &Cx, round: u32, need: &QuicNeedMore) {
let round = round.to_string();
let pending = need.pending.len().to_string();
let repair_blocks = need.repair_blocks.len().to_string();
let repair_symbols = need
.repair_blocks
.iter()
.fold(0u64, |acc, request| {
acc.saturating_add(u64::from(request.symbols))
})
.to_string();
let source_symbols = need.source_symbols.len().to_string();
let round_symbols_observed = need.round_symbols_observed.unwrap_or(0).to_string();
let round_symbols_accepted = need.round_symbols_accepted.unwrap_or(0).to_string();
let round_loss_fraction = format!("{:.4}", need.round_loss_fraction.unwrap_or(0.0));
let pending_rank = need.pending_rank.unwrap_or(0).to_string();
let pending_rank_columns = need.pending_rank_columns.unwrap_or(0).to_string();
let pending_rank_deficit = need.pending_rank_deficit.unwrap_or(0).to_string();
let pending_decode_jobs = need.pending_decode_jobs.unwrap_or(0).to_string();
// LogEntry caps fields at MAX_FIELDS (16): everything past the 16th
// field was silently ignored here, and prioritized correlation ids
// (task/region/span) evict the leading fields of a full entry. Keep
// this emission at <=12 explicit fields; the cumulative intake stats
// (drain/pump/staging) already ship on the dedicated
// "atp_quic.receive.intake" entry from trace_summary.
cx.trace_with_fields(
"atp_quic.receive.need_more",
&[
("round", round.as_str()),
("pending", pending.as_str()),
("repair_blocks", repair_blocks.as_str()),
("repair_symbols", repair_symbols.as_str()),
("source_symbols", source_symbols.as_str()),
("round_symbols_observed", round_symbols_observed.as_str()),
("round_symbols_accepted", round_symbols_accepted.as_str()),
("round_loss_fraction", round_loss_fraction.as_str()),
("pending_rank", pending_rank.as_str()),
("pending_rank_columns", pending_rank_columns.as_str()),
("pending_rank_deficit", pending_rank_deficit.as_str()),
("pending_decode_jobs", pending_decode_jobs.as_str()),
],
);
}
fn trace_summary(&self, cx: &Cx, transfer_id: &str) {
let drain_calls = self.drain_calls.to_string();
let symbols_observed = self.symbols_observed.to_string();
let symbols_accepted = self.symbols_accepted.to_string();
let blocks_completed = self.blocks_completed.to_string();
let drain_micros = self.drain_micros.to_string();
let pump_calls = self.pump_calls.to_string();
let pump_packets = self.pump_packets.to_string();
let pump_micros = self.pump_micros.to_string();
let staging_write_count = self.staging_write_count.to_string();
let staging_write_bytes = self.staging_write_bytes.to_string();
let staging_write_micros = self.staging_write_micros.to_string();
cx.trace_with_fields(
"atp_quic.receive.intake",
&[
("transfer_id", transfer_id),
("drain_calls", drain_calls.as_str()),
("symbols_observed", symbols_observed.as_str()),
("symbols_accepted", symbols_accepted.as_str()),
("blocks_completed", blocks_completed.as_str()),
("drain_micros", drain_micros.as_str()),
("pump_calls", pump_calls.as_str()),
("pump_packets", pump_packets.as_str()),
("pump_micros", pump_micros.as_str()),
("staging_write_count", staging_write_count.as_str()),
("staging_write_bytes", staging_write_bytes.as_str()),
("staging_write_micros", staging_write_micros.as_str()),
],
);
}
}
/// One planned member write for a packed entry commit.
struct QuicPackedMemberWrite {
offset: u64,
len: u64,
staging_path: PathBuf,
out_path: PathBuf,
/// Member metadata applied inside the one-shot batch task (sync core) so
/// a 2000-member tree pays zero per-file pool dispatches for metadata.
metadata: Option<EntryMetadata>,
}
struct QuicPackedStagingGuard {
path: PathBuf,
staging_root: PathBuf,
armed: bool,
}
impl QuicPackedStagingGuard {
fn new(path: PathBuf, staging_root: PathBuf) -> Self {
Self {
path,
staging_root,
armed: true,
}
}
fn disarm(&mut self) {
self.armed = false;
}
}
impl Drop for QuicPackedStagingGuard {
fn drop(&mut self) {
if self.armed {
let _ = std::fs::remove_file(&self.path);
// A dropped blocking future can race the outer directory guard.
// Once the batch closure returns, this second owned cleanup runs
// after all source/output handles are closed.
let _ = std::fs::remove_dir_all(&self.staging_root);
}
}
}
struct QuicPackedBatchWrite {
reports: Vec<(PathBuf, MetadataApplyReport)>,
guards: Vec<QuicPackedStagingGuard>,
metadata_deferred: Vec<bool>,
}
/// One-shot packed-member commit cap (mirrors the RQ tier): above this staged
/// span the batch falls back to the streaming cursor loop.
const QUIC_PACKED_MEMBER_BATCH_ONESHOT_MAX_BYTES: u64 = 128 * 1024 * 1024;
/// Commit an entire packed small-file batch inside ONE blocking-pool task:
/// read the verified staged span once, then create/write every member with
/// raw `std::fs`. The serial async loop pays several pool round-trips per
/// tiny file — the dispatch-overhead commit tail that MATRIX-211 eliminated
/// on the RQ tier.
fn write_quic_packed_member_batch_oneshot(
staging_path: PathBuf,
members: Vec<QuicPackedMemberWrite>,
span_len: usize,
sparse_files: bool,
) -> std::io::Result<QuicPackedBatchWrite> {
use std::io::{Read as _, Write as _};
let staging_root = staging_path
.parent()
.ok_or_else(|| std::io::Error::other("packed staging path has no parent"))?
.to_path_buf();
let mut reports = Vec::new();
let mut guards = Vec::with_capacity(members.len());
let mut metadata_deferred = Vec::with_capacity(members.len());
let operation = (|| -> std::io::Result<()> {
let mut staged = vec![0u8; span_len];
let mut source = std::fs::File::open(&staging_path)?;
source.read_exact(&mut staged)?;
for member in &members {
let start = usize::try_from(member.offset)
.map_err(|_| std::io::Error::other("packed member offset exceeds span"))?;
let len = usize::try_from(member.len)
.map_err(|_| std::io::Error::other("packed member length exceeds span"))?;
let end = start
.checked_add(len)
.filter(|end| *end <= staged.len())
.ok_or_else(|| std::io::Error::other("packed member range exceeds staged span"))?;
let mut out = std::fs::OpenOptions::new()
.write(true)
.create_new(true)
.open(&member.staging_path)?;
let guard =
QuicPackedStagingGuard::new(member.staging_path.clone(), staging_root.clone());
if sparse_files {
out.set_len(member.len)?;
crate::net::atp::transport_common::metadata::write_sparse_zero_runs_sync(
&mut out,
&staged[start..end],
)?;
} else {
out.write_all(&staged[start..end])?;
}
out.sync_all()?;
drop(out);
let deferred = member
.metadata
.as_ref()
.is_some_and(super::metadata_makes_windows_staging_readonly);
if let Some(metadata) = &member.metadata
&& !deferred
{
let report =
super::apply_quic_entry_metadata_sync_verified(&member.staging_path, metadata)
.map_err(|err| std::io::Error::other(err.into_message()))?;
reports.push((member.out_path.clone(), report));
}
metadata_deferred.push(deferred);
guards.push(guard);
}
Ok(())
})();
if let Err(error) = operation {
drop(guards);
return Err(error);
}
Ok(QuicPackedBatchWrite {
reports,
guards,
metadata_deferred,
})
}
fn write_quic_packed_member_batch_streaming(
staging_path: PathBuf,
members: Vec<QuicPackedMemberWrite>,
sparse_files: bool,
) -> std::io::Result<QuicPackedBatchWrite> {
use std::io::{Read as _, Seek as _, Write as _};
let staging_root = staging_path
.parent()
.ok_or_else(|| std::io::Error::other("packed staging path has no parent"))?
.to_path_buf();
let mut reports = Vec::new();
let mut guards = Vec::with_capacity(members.len());
let mut metadata_deferred = Vec::with_capacity(members.len());
let operation = (|| -> std::io::Result<()> {
let mut source = std::fs::File::open(&staging_path)?;
for member in &members {
source.seek(std::io::SeekFrom::Start(member.offset))?;
let mut out = std::fs::OpenOptions::new()
.write(true)
.create_new(true)
.open(&member.staging_path)?;
let guard =
QuicPackedStagingGuard::new(member.staging_path.clone(), staging_root.clone());
let copied = if sparse_files {
out.set_len(member.len)?;
let mut remaining = member.len;
let mut copied = 0u64;
let mut buffer = vec![0u8; super::QUIC_SOURCE_STREAM_READ_CHUNK];
while remaining != 0 {
let limit = usize::try_from(remaining)
.unwrap_or(usize::MAX)
.min(buffer.len());
let read = source.read(&mut buffer[..limit])?;
if read == 0 {
break;
}
crate::net::atp::transport_common::metadata::write_sparse_zero_runs_sync(
&mut out,
&buffer[..read],
)?;
let read = u64::try_from(read).unwrap_or(u64::MAX);
copied = copied.saturating_add(read);
remaining = remaining.saturating_sub(read);
}
copied
} else {
std::io::copy(
&mut std::io::Read::by_ref(&mut source).take(member.len),
&mut out,
)?
};
if copied != member.len {
return Err(std::io::Error::other(format!(
"short read while splitting packed member {}",
member.out_path.display()
)));
}
out.flush()?;
out.sync_all()?;
drop(out);
let deferred = member
.metadata
.as_ref()
.is_some_and(super::metadata_makes_windows_staging_readonly);
if let Some(metadata) = &member.metadata
&& !deferred
{
let report =
super::apply_quic_entry_metadata_sync_verified(&member.staging_path, metadata)
.map_err(|err| std::io::Error::other(err.into_message()))?;
reports.push((member.out_path.clone(), report));
}
metadata_deferred.push(deferred);
guards.push(guard);
}
Ok(())
})();
if let Err(error) = operation {
drop(guards);
return Err(error);
}
Ok(QuicPackedBatchWrite {
reports,
guards,
metadata_deferred,
})
}
/// Split a verified staged pack into guarded member staging files.
async fn write_quic_packed_member_batch(
staging_path: &Path,
members: &[QuicPackedMemberWrite],
sparse_files: bool,
) -> Result<QuicPackedBatchWrite, QuicTransportError> {
let span_end = members
.iter()
.map(|member| member.offset.saturating_add(member.len))
.max()
.unwrap_or(0);
if members.len() > 1
&& span_end <= QUIC_PACKED_MEMBER_BATCH_ONESHOT_MAX_BYTES
&& let Ok(span_len) = usize::try_from(span_end)
{
let staging = staging_path.to_path_buf();
let batch = members
.iter()
.map(|member| QuicPackedMemberWrite {
offset: member.offset,
len: member.len,
staging_path: member.staging_path.clone(),
out_path: member.out_path.clone(),
metadata: member.metadata.clone(),
})
.collect::<Vec<_>>();
let staging_display = staging_path.display().to_string();
return crate::runtime::spawn_blocking_io(move || {
write_quic_packed_member_batch_oneshot(staging, batch, span_len, sparse_files)
})
.await
.map_err(|err| QuicTransportError::Source(format!("{staging_display}: {err}")));
}
let staging = staging_path.to_path_buf();
let batch = members
.iter()
.map(|member| QuicPackedMemberWrite {
offset: member.offset,
len: member.len,
staging_path: member.staging_path.clone(),
out_path: member.out_path.clone(),
metadata: member.metadata.clone(),
})
.collect::<Vec<_>>();
let staging_display = staging_path.display().to_string();
crate::runtime::spawn_blocking_io(move || {
write_quic_packed_member_batch_streaming(staging, batch, sparse_files)
})
.await
.map_err(|err| QuicTransportError::Source(format!("{staging_display}: {err}")))
}
/// Verify each packed member's SHA-256 against the staged pack and append the
/// logical (per-member) digests the merkle check needs. Returns whether every
/// member hash matched.
async fn hash_quic_packed_members_streaming(
staging_path: &Path,
members: &[crate::net::atp::transport_tcp::PackedMember],
digests: &mut Vec<EntryDigest>,
read_buf: &mut [u8],
) -> Result<bool, QuicTransportError> {
let mut file = crate::fs::File::open(staging_path)
.await
.map_err(|err| QuicTransportError::Source(format!("{}: {err}", staging_path.display())))?;
let mut cursor = 0u64;
let mut members_ok = true;
for member in members {
if cursor != member.offset {
file.seek(std::io::SeekFrom::Start(member.offset)).await?;
cursor = member.offset;
}
let mut sha = Sha256::new();
let mut content_id = crate::atp::object::ContentId::streaming();
let mut remaining = member.len;
while remaining > 0 {
let want =
usize::try_from(remaining.min(read_buf.len() as u64)).unwrap_or(read_buf.len());
let n = file.read(&mut read_buf[..want]).await.map_err(|err| {
QuicTransportError::Source(format!("{}: {err}", staging_path.display()))
})?;
if n == 0 {
return Err(QuicTransportError::Source(format!(
"{}: short read while verifying packed member {}",
staging_path.display(),
member.rel_path
)));
}
sha.update(&read_buf[..n]);
content_id.update(&read_buf[..n]);
remaining -= n as u64;
cursor = cursor.saturating_add(n as u64);
}
let member_sha: [u8; 32] = sha.finalize().into();
if hex_encode(&member_sha) != member.sha256_hex {
members_ok = false;
}
digests.push(EntryDigest {
rel_path: member.rel_path.clone(),
size: member.len,
content_id: crate::atp::object::ObjectId::content(content_id.finalize()),
content_sha256: member_sha,
});
}
Ok(members_ok)
}
async fn commit_staged_entries(
cx: &Cx,
link: &mut QuicLink,
control: &mut NativeQuicFrameTransport,
dest_dir: &Path,
manifest: &TransferManifest,
staged: &mut [QuicStagedEntryReceive],
config: &QuicConfig,
options: &QuicReceiveOptions,
) -> Result<(ReceiveReceipt, Vec<PathBuf>), QuicTransportError> {
let mut read_buf = vec![0_u8; config.chunk_size.max(1)];
let mut sha_ok = true;
let mut digests = Vec::with_capacity(manifest.entries.len());
trace_blocking_roundtrip_probe("commit_start").await;
for (entry, staged_entry) in manifest.entries.iter().zip(staged.iter_mut()) {
cx.checkpoint().map_err(|_| QuicTransportError::Cancelled)?;
let verify_started = Instant::now();
send_and_flush_native_keep_alive(cx, link, control).await?;
staged_entry.close_cached_staging_file().await?;
staged_entry.ensure_created(entry).await?;
let (size, content_id, content_sha256) =
match staged_entry.take_inline_hash_if_complete(entry.size) {
Some(inline) => inline,
None => hash_file_streaming(&staged_entry.staging_path, &mut read_buf).await?,
};
if size != entry.size || hex_encode(&content_sha256) != entry.sha256_hex {
sha_ok = false;
}
if entry.members.is_empty() {
digests.push(EntryDigest {
rel_path: entry.rel_path.clone(),
size,
content_id,
content_sha256,
});
} else if !hash_quic_packed_members_streaming(
&staged_entry.staging_path,
&entry.members,
&mut digests,
&mut read_buf,
)
.await?
{
// Logical digests replace the pack digest for the merkle check;
// any member hash mismatch fails the transfer closed.
sha_ok = false;
}
send_and_flush_native_keep_alive(cx, link, control).await?;
quic_rqtrace!(
"receiver: verify_entry index={} size={} members={} elapsed_ms={}",
entry.index,
entry.size,
entry.members.len(),
verify_started.elapsed().as_millis()
);
}
let merkle_ok = flat_merkle_root_from_digests(&digests) == manifest.merkle_root_hex;
let metadata_ok = super::manifest_metadata_commitment(manifest) == manifest.metadata_root_hex;
let committed = sha_ok && merkle_ok && metadata_ok;
let mut committed_paths = Vec::new();
if committed {
super::prepare_quic_destination_root(dest_dir).await?;
let base = super::quic_safe_base_for_root_name(dest_dir, &manifest.root_name)?;
if manifest.is_directory && manifest.entries.is_empty() {
super::reject_quic_destination_symlink_prefix(&base, &base).await?;
crate::fs::create_dir_all(&base).await?;
super::reject_quic_destination_symlink_prefix(&base, &base).await?;
committed_paths.push(base.clone());
send_and_flush_native_keep_alive(cx, link, control).await?;
}
let mut commit_clock = Instant::now();
for (entry, staged_entry) in manifest.entries.iter().zip(staged.iter_mut()) {
cx.checkpoint().map_err(|_| QuicTransportError::Cancelled)?;
quic_rqtrace!(
"receiver: commit_entry index={} size={} members={} since_prev_ms={}",
entry.index,
entry.size,
entry.members.len(),
commit_clock.elapsed().as_millis()
);
commit_clock = Instant::now();
send_and_flush_native_keep_alive(cx, link, control).await?;
staged_entry.close_cached_staging_file().await?;
if !entry.members.is_empty() {
// Packed entry: split the verified staged pack into member
// files AND apply member metadata with one blocking-pool
// batch (the staged pack itself is scratch; the staging-dir
// guard reclaims it).
let mut writes = Vec::with_capacity(entry.members.len());
for (member_index, member) in entry.members.iter().enumerate() {
let member_path = super::quic_join_relative(&base, &member.rel_path)?;
super::reject_quic_destination_symlink_prefix(&base, &member_path).await?;
writes.push(QuicPackedMemberWrite {
offset: member.offset,
len: member.len,
staging_path: staged_entry
.staging_path
.with_file_name(format!("member-{}-{member_index}", entry.index)),
out_path: member_path,
metadata: member.metadata.clone(),
});
}
let mut batch = keep_peer_alive_while(
cx,
link,
control,
write_quic_packed_member_batch(
&staged_entry.staging_path,
&writes,
options.sparse_files(),
),
)
.await?;
for (path, report) in &batch.reports {
super::trace_quic_metadata_skips(cx, path, report);
}
let mut member_clock = Instant::now();
let mut member_keep_alive = LocalWorkKeepAlive::new(Instant::now());
for (member_index, (member, write)) in entry.members.iter().zip(&writes).enumerate()
{
keep_peer_alive_during_local_work(cx, link, control, &mut member_keep_alive)
.await?;
if member_index % 250 == 0 {
quic_rqtrace!(
"receiver: commit_member entry={} index={member_index} of={} since_prev_ms={}",
entry.index,
entry.members.len(),
member_clock.elapsed().as_millis()
);
member_clock = Instant::now();
trace_blocking_roundtrip_probe("commit_member").await;
}
if let Some(parent) = write.out_path.parent() {
crate::fs::create_dir_all(parent).await?;
}
super::reject_quic_destination_symlink_prefix(&base, &write.out_path).await?;
crate::net::atp::transport_common::metadata::commit_staged_regular_file_transactionally(
&write.staging_path,
&write.out_path,
)
.await?;
batch.guards[member_index].disarm();
if batch.metadata_deferred[member_index] {
super::apply_quic_member_metadata(cx, &write.out_path, member).await?;
}
committed_paths.push(write.out_path.clone());
}
send_and_flush_native_keep_alive(cx, link, control).await?;
continue;
}
let out_path = if manifest.is_directory {
super::quic_join_relative(&base, &entry.rel_path)?
} else {
base.clone()
};
match super::commit_quic_metadata_entry(cx, &base, &out_path, entry, config).await? {
super::QuicMetadataCommit::Committed => {
committed_paths.push(out_path);
send_and_flush_native_keep_alive(cx, link, control).await?;
continue;
}
super::QuicMetadataCommit::Skipped => continue,
super::QuicMetadataCommit::Regular => {}
}
super::reject_quic_destination_symlink_prefix(&base, &out_path).await?;
if let Some(parent) = out_path.parent() {
crate::fs::create_dir_all(parent).await?;
}
let metadata_deferred = entry
.metadata
.as_ref()
.is_some_and(super::metadata_makes_windows_staging_readonly);
if !metadata_deferred {
super::apply_quic_entry_metadata(cx, &staged_entry.staging_path, entry).await?;
}
super::reject_quic_destination_symlink_prefix(&base, &out_path).await?;
crate::net::atp::transport_common::metadata::commit_staged_regular_file_transactionally(
&staged_entry.staging_path,
&out_path,
)
.await?;
if metadata_deferred {
super::apply_quic_entry_metadata(cx, &out_path, entry).await?;
}
committed_paths.push(out_path);
send_and_flush_native_keep_alive(cx, link, control).await?;
}
super::apply_quic_directory_metadata(cx, &base, manifest).await?;
send_and_flush_native_keep_alive(cx, link, control).await?;
}
let bytes_received = digests
.iter()
.fold(0u64, |acc, digest| acc.saturating_add(digest.size));
// Logical file count: packed entries contribute one per member.
let logical_files = manifest.entries.iter().fold(0usize, |acc, entry| {
acc.saturating_add(entry.members.len().max(1))
});
super::quic_progress(format_args!(
"receiver: commit_decision transfer={} committed={} sha_ok={} merkle_ok={} metadata_ok={} bytes_received={} files={}",
manifest.transfer_id,
committed,
sha_ok,
merkle_ok,
metadata_ok,
bytes_received,
logical_files
));
Ok((
ReceiveReceipt {
committed,
bytes_received,
files: u32::try_from(logical_files).unwrap_or(u32::MAX),
sha_ok,
merkle_ok,
symbols_accepted: 0,
feedback_rounds: 0,
decode_count: 0,
decode_micros: 0,
reason: if committed {
None
} else if !sha_ok {
Some("per-entry SHA-256 mismatch".to_string())
} else if !merkle_ok {
Some("merkle-root mismatch".to_string())
} else {
Some("metadata commitment mismatch".to_string())
},
committed_paths: committed_paths
.iter()
.map(|path| path.display().to_string())
.collect(),
},
committed_paths,
))
}
#[allow(dead_code)]
async fn read_native_source_stream_chunk(
cx: &Cx,
link: &mut QuicLink,
stream: StreamId,
max_len: usize,
idle_timeout: Duration,
) -> Result<Bytes, QuicTransportError> {
loop {
match link.conn.read_stream_bytes(cx, stream, max_len) {
Ok(chunk) => {
if !chunk.is_empty() || link.conn.is_stream_read_eof(stream)? {
return Ok(chunk);
}
}
Err(NativeQuicConnectionError::StreamTable(StreamTableError::UnknownStream(
unknown,
))) if unknown == stream => {}
Err(err) => return Err(err.into()),
}
// Small drain budget: the read loop consumes the reassembly map
// between pumps, so bounding each pump turn bounds receiver backlog
// RSS instead of letting the pump ingest an entire drain-until-empty
// budget ahead of the staging writer.
if link
.pump_inbound_for_with_drain_budget(
cx,
idle_timeout,
QUIC_SOURCE_STREAM_READ_DRAIN_BATCHES,
)
.await?
== 0
{
return Err(QuicTransportError::Quic(format!(
"transport timeout during receive QUIC source stream after {idle_timeout:?}; \
udp_packets_received={} one_rtt_packets_ingested={} \
unprotect_packets_dropped={} non_one_rtt_packets_dropped={} \
pending_inbound_packets={} stream_read_eof={:?}",
link.udp_packets_received,
link.one_rtt_packets_ingested,
link.unprotect_packets_dropped,
link.non_one_rtt_packets_dropped,
link.pending_inbound_packets(),
link.conn.is_stream_read_eof(stream),
)));
}
let _ = link.flush(cx).await?;
}
}
#[allow(dead_code)]
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
struct SourceSymbolKey {
entry: u32,
sbn: u8,
esi: u32,
}
#[allow(dead_code)]
#[derive(Debug, Clone, Copy)]
struct SourceStreamEntryPlan {
entry: u32,
stream_start: u64,
size: u64,
}
#[allow(dead_code)]
impl SourceStreamEntryPlan {
fn stream_end(self) -> u64 {
self.stream_start.saturating_add(self.size)
}
}
#[allow(dead_code)]
struct PartialSourceSymbol {
data: Vec<u8>,
received: Vec<bool>,
received_count: usize,
}
#[allow(dead_code)]
impl PartialSourceSymbol {
fn new(symbol_size: usize, expected_len: usize) -> Self {
Self {
data: vec![0; symbol_size],
received: vec![false; expected_len],
received_count: 0,
}
}
fn push_fragment(
&mut self,
offset: usize,
fragment: &[u8],
) -> Result<bool, QuicTransportError> {
if offset.saturating_add(fragment.len()) > self.received.len() {
return Err(QuicTransportError::Integrity(
"source stream frame crossed a RaptorQ source-symbol boundary".to_string(),
));
}
for (idx, byte) in fragment.iter().enumerate() {
let pos = offset + idx;
if !self.received[pos] {
self.data[pos] = *byte;
self.received[pos] = true;
self.received_count = self.received_count.saturating_add(1);
}
}
Ok(self.received_count == self.received.len())
}
}
#[allow(dead_code)]
struct NativeSourceStreamSymbolIntake {
plans: Vec<SourceStreamEntryPlan>,
partials: BTreeMap<SourceSymbolKey, PartialSourceSymbol>,
fed: BTreeSet<SourceSymbolKey>,
total_bytes: u64,
fin_seen: bool,
}
#[allow(dead_code)]
impl NativeSourceStreamSymbolIntake {
fn new(manifest: &TransferManifest) -> Result<Self, QuicTransportError> {
let mut stream_start = 0u64;
let mut plans = Vec::with_capacity(manifest.entries.len());
for entry in &manifest.entries {
plans.push(SourceStreamEntryPlan {
entry: entry.index,
stream_start,
size: entry.size,
});
stream_start = stream_start.checked_add(entry.size).ok_or_else(|| {
QuicTransportError::TooLarge {
size: manifest.total_bytes,
max: u64::MAX,
}
})?;
}
if stream_start != manifest.total_bytes {
return Err(QuicTransportError::Integrity(format!(
"source stream manifest total {} did not match entry sum {stream_start}",
manifest.total_bytes
)));
}
Ok(Self {
plans,
partials: BTreeMap::new(),
fed: BTreeSet::new(),
total_bytes: manifest.total_bytes,
fin_seen: false,
})
}
fn fin_seen(&self) -> bool {
self.fin_seen
}
fn feed_stream_frame(
&mut self,
cx: &Cx,
decoders: &mut [super::QuicEntryDecoder],
config: &QuicConfig,
decode_stats: &mut super::QuicDecodeStats,
completed_backlog: &mut Vec<super::QuicDecodedBlock>,
frame: &ReceivedSourceStreamFrame,
) -> Result<u64, QuicTransportError> {
let frame_len = u64::try_from(frame.data.len()).unwrap_or(u64::MAX);
let frame_end = frame.offset.checked_add(frame_len).ok_or_else(|| {
QuicTransportError::Integrity("source stream frame offset overflow".to_string())
})?;
if frame_end > self.total_bytes {
return Err(QuicTransportError::Integrity(format!(
"source stream carried bytes beyond the manifest total: end={frame_end}, total={}",
self.total_bytes
)));
}
if frame.fin {
if frame_end != self.total_bytes {
return Err(QuicTransportError::Integrity(format!(
"source stream FIN at offset {frame_end}, expected {}",
self.total_bytes
)));
}
self.fin_seen = true;
}
if frame.data.is_empty() {
return Ok(0);
}
self.feed_global_bytes(
cx,
decoders,
config,
decode_stats,
completed_backlog,
frame.offset,
frame.data.as_ref(),
)
}
fn feed_global_bytes(
&mut self,
cx: &Cx,
decoders: &mut [super::QuicEntryDecoder],
config: &QuicConfig,
decode_stats: &mut super::QuicDecodeStats,
completed_backlog: &mut Vec<super::QuicDecodedBlock>,
stream_offset: u64,
bytes: &[u8],
) -> Result<u64, QuicTransportError> {
let mut accepted = 0u64;
let mut consumed = 0usize;
while consumed < bytes.len() {
let absolute = stream_offset
.checked_add(u64::try_from(consumed).unwrap_or(u64::MAX))
.ok_or_else(|| {
QuicTransportError::Integrity("source stream offset overflow".to_string())
})?;
let Some(plan) = self
.plans
.iter()
.copied()
.find(|plan| absolute >= plan.stream_start && absolute < plan.stream_end())
else {
return Err(QuicTransportError::Integrity(format!(
"source stream frame offset {absolute} outside manifest entries"
)));
};
let entry_offset = absolute.saturating_sub(plan.stream_start);
let available_in_entry =
usize::try_from(plan.stream_end().saturating_sub(absolute)).unwrap_or(usize::MAX);
let take = (bytes.len() - consumed).min(available_in_entry);
accepted = accepted.saturating_add(self.feed_entry_bytes(
cx,
decoders,
config,
decode_stats,
completed_backlog,
plan,
entry_offset,
&bytes[consumed..consumed + take],
)?);
consumed += take;
}
Ok(accepted)
}
fn feed_entry_bytes(
&mut self,
cx: &Cx,
decoders: &mut [super::QuicEntryDecoder],
config: &QuicConfig,
decode_stats: &mut super::QuicDecodeStats,
completed_backlog: &mut Vec<super::QuicDecodedBlock>,
plan: SourceStreamEntryPlan,
entry_offset: u64,
bytes: &[u8],
) -> Result<u64, QuicTransportError> {
let symbol_size = usize::from(config.symbol_size.max(1));
let symbol_size_u64 = u64::try_from(symbol_size).unwrap_or(u64::MAX);
let max_block_size = u64::try_from(config.max_block_size.max(1)).unwrap_or(u64::MAX);
let mut accepted = 0u64;
let mut consumed = 0usize;
while consumed < bytes.len() {
let offset = entry_offset
.checked_add(u64::try_from(consumed).unwrap_or(u64::MAX))
.ok_or_else(|| {
QuicTransportError::Integrity("entry source stream offset overflow".to_string())
})?;
let block_index = offset / max_block_size;
let sbn = u8::try_from(block_index).map_err(|_| QuicTransportError::TooLarge {
size: plan.size,
max: max_block_size.saturating_mul(u64::from(u8::MAX) + 1),
})?;
let block_start = block_index.saturating_mul(max_block_size);
let block_len = plan.size.saturating_sub(block_start).min(max_block_size);
let block_offset = offset.saturating_sub(block_start);
let esi_u64 = block_offset / symbol_size_u64;
let esi = u32::try_from(esi_u64).map_err(|_| {
QuicTransportError::Integrity("source symbol ESI exceeded u32 range".to_string())
})?;
let symbol_offset_u64 = block_offset % symbol_size_u64;
let symbol_start = esi_u64.saturating_mul(symbol_size_u64);
let expected_len = usize::try_from(
block_len
.saturating_sub(symbol_start)
.min(symbol_size_u64)
.max(1),
)
.unwrap_or(usize::MAX);
let symbol_offset = usize::try_from(symbol_offset_u64).unwrap_or(usize::MAX);
let available_in_symbol = expected_len.saturating_sub(symbol_offset);
let take = (bytes.len() - consumed).min(available_in_symbol);
if take == 0 {
return Err(QuicTransportError::Integrity(
"source stream symbol split made no progress".to_string(),
));
}
accepted = accepted.saturating_add(self.feed_symbol_fragment(
cx,
decoders,
config,
decode_stats,
completed_backlog,
SourceSymbolKey {
entry: plan.entry,
sbn,
esi,
},
symbol_size,
expected_len,
symbol_offset,
&bytes[consumed..consumed + take],
)?);
consumed += take;
}
Ok(accepted)
}
fn feed_symbol_fragment(
&mut self,
cx: &Cx,
decoders: &mut [super::QuicEntryDecoder],
config: &QuicConfig,
decode_stats: &mut super::QuicDecodeStats,
completed_backlog: &mut Vec<super::QuicDecodedBlock>,
key: SourceSymbolKey,
symbol_size: usize,
expected_len: usize,
symbol_offset: usize,
fragment: &[u8],
) -> Result<u64, QuicTransportError> {
if self.fed.contains(&key) {
return Ok(0);
}
let complete = {
let partial = self
.partials
.entry(key)
.or_insert_with(|| PartialSourceSymbol::new(symbol_size, expected_len));
partial.push_fragment(symbol_offset, fragment)?
};
if !complete {
return Ok(0);
}
let partial = self.partials.remove(&key).ok_or_else(|| {
QuicTransportError::Integrity("completed source symbol disappeared".to_string())
})?;
self.fed.insert(key);
let decoder_index = decoders
.iter()
.position(|decoder| decoder.index == key.entry)
.ok_or_else(|| {
QuicTransportError::Integrity(format!(
"source stream symbol for unknown manifest entry {}",
key.entry
))
})?;
let symbol = Symbol::new(
SymbolId::new(decoders[decoder_index].object_id, key.sbn, key.esi),
partial.data,
SymbolKind::Source,
);
let transfer_decode_width = super::quic_transfer_decode_width(decoders, config);
let allow_spawn_decode = super::quic_pending_decode_jobs(decoders) < transfer_decode_width;
let (accepted, decoded) = super::feed_authenticated_symbol_take_block_deferred(
cx,
&mut decoders[decoder_index],
AuthenticatedSymbol::new_unauthenticated(symbol),
config,
decode_stats,
allow_spawn_decode,
transfer_decode_width,
)?;
if let Some(block) = decoded {
completed_backlog.push(block);
}
Ok(u64::from(accepted))
}
}
#[allow(dead_code)]
async fn drain_native_source_stream_tap_frames(
cx: &Cx,
link: &mut QuicLink,
intake: &mut NativeSourceStreamSymbolIntake,
decoders: &mut [super::QuicEntryDecoder],
config: &QuicConfig,
decode_stats: &mut super::QuicDecodeStats,
completed_backlog: &mut Vec<super::QuicDecodedBlock>,
) -> Result<(usize, u64), QuicTransportError> {
let frames = link.drain_received_source_stream_frames();
let mut accepted = 0u64;
for frame in &frames {
accepted = accepted.saturating_add(intake.feed_stream_frame(
cx,
decoders,
config,
decode_stats,
completed_backlog,
frame,
)?);
}
if !frames.is_empty() {
let transfer_decode_width = super::quic_transfer_decode_width(decoders, config);
let allow_spawn_decode = super::quic_pending_decode_jobs(decoders) < transfer_decode_width;
completed_backlog.extend(
super::drain_ready_quic_decodes_with_blocks(
cx,
decoders,
config,
decode_stats,
allow_spawn_decode,
transfer_decode_width,
)
.await?,
);
}
Ok((frames.len(), accepted))
}
fn validate_native_source_stream_observed_completion(
link: &mut QuicLink,
expected_total: u64,
) -> Result<bool, QuicTransportError> {
let observed_end = link.source_stream_observed_end;
if observed_end > expected_total {
return Err(QuicTransportError::Integrity(format!(
"source stream carried bytes beyond the manifest total: end={observed_end}, total={expected_total}"
)));
}
match link.source_stream_observed_fin_end {
Some(fin_end) if fin_end != expected_total => Err(QuicTransportError::Integrity(format!(
"source stream FIN at offset {fin_end}, expected {expected_total}"
))),
Some(_) => Ok(true),
None => Ok(false),
}
}
async fn receive_native_source_stream_entries_pumped(
cx: &Cx,
link: &mut QuicLink,
stream: StreamId,
manifest: &TransferManifest,
staged: &mut [QuicStagedEntryReceive],
config: &QuicConfig,
) -> Result<(u64, u64, super::QuicDecodeStats), QuicTransportError> {
let mut received = 0u64;
// Profiling-only (env ATP_QUIC_RECV_PROFILE): attribute the source-stream receiver's
// per-byte CPU between pump+AEAD-decrypt (read chunk), staging write, and per-chunk
// ACK flush — to target the cross-core parallelization for encrypted-large throughput.
// No behavior change when unset.
let recv_profile = std::env::var_os("ATP_QUIC_RECV_PROFILE").is_some();
let (mut read_us, mut write_us, mut flush_us) = (0u64, 0u64, 0u64);
for (entry_index, entry) in manifest.entries.iter().enumerate() {
cx.checkpoint().map_err(|_| QuicTransportError::Cancelled)?;
let staged_entry = staged.get_mut(entry_index).ok_or_else(|| {
QuicTransportError::Integrity(format!(
"source stream has no staging slot for entry {}",
entry.index
))
})?;
let mut entry_offset = 0u64;
while entry_offset < entry.size {
cx.checkpoint().map_err(|_| QuicTransportError::Cancelled)?;
let remaining = entry.size.saturating_sub(entry_offset);
let chunk_len = usize::try_from(
remaining
.min(u64::try_from(config.chunk_size.max(1)).unwrap_or(u64::MAX))
.min(u64::try_from(super::QUIC_SOURCE_STREAM_READ_CHUNK).unwrap_or(u64::MAX)),
)
.unwrap_or(super::QUIC_SOURCE_STREAM_READ_CHUNK)
.max(1);
let read_at = recv_profile.then(Instant::now);
let chunk =
read_native_source_stream_chunk(cx, link, stream, chunk_len, config.idle_timeout)
.await?;
if let Some(t) = read_at {
read_us = read_us.saturating_add(t.elapsed().as_micros() as u64);
}
if chunk.is_empty() {
return Err(QuicTransportError::Integrity(format!(
"source stream ended before entry {} completed ({} of {} bytes)",
entry.index, entry_offset, entry.size
)));
}
let write_at = recv_profile.then(Instant::now);
staged_entry
.write_range(entry, entry_offset, chunk.as_ref())
.await?;
if let Some(t) = write_at {
write_us = write_us.saturating_add(t.elapsed().as_micros() as u64);
}
let n = u64::try_from(chunk.len()).unwrap_or(u64::MAX);
entry_offset = entry_offset.checked_add(n).ok_or_else(|| {
QuicTransportError::Integrity(format!(
"source stream entry {} byte count overflow",
entry.index
))
})?;
received = received.saturating_add(n);
let flush_at = recv_profile.then(Instant::now);
let chunk_flushed = link.flush(cx).await?;
if let Some(t) = flush_at {
flush_us = flush_us.saturating_add(t.elapsed().as_micros() as u64);
}
quic_rqtrace!(
"receiver: source_chunk entry={} end_offset={entry_offset} len={n} flush_packets={chunk_flushed}",
entry.index
);
}
}
if recv_profile {
eprintln!(
"[atp-quic-recv-profile] read_pump_decrypt_micros={read_us} staging_write_micros={write_us} per_chunk_flush_ack_micros={flush_us} total_bytes={received}"
);
}
let extra = link.conn.read_stream_bytes(cx, stream, 1)?;
if !extra.is_empty() {
return Err(QuicTransportError::Integrity(
"source stream carried bytes beyond the manifest total".to_string(),
));
}
let fin_seen = validate_native_source_stream_observed_completion(link, manifest.total_bytes)?;
if received != manifest.total_bytes {
return Err(QuicTransportError::Integrity(format!(
"source stream delivered {received} bytes, expected {}",
manifest.total_bytes
)));
}
if !fin_seen && !link.conn.is_stream_read_eof(stream)? {
quic_rqtrace!(
"receiver: native_source_stream byte-complete before FIN bytes={} stream={}",
received,
stream.0
);
}
flush_cached_quic_staging_files(staged).await?;
Ok((received, 0, super::QuicDecodeStats::default()))
}
async fn drain_native_receiver_symbol_queue(
cx: &Cx,
link: &mut QuicLink,
manifest: &TransferManifest,
decoders: &mut [super::QuicEntryDecoder],
config: &QuicConfig,
decode_stats: &mut super::QuicDecodeStats,
intake_stats: &mut NativeReceiverIntakeStats,
completed_backlog: &mut Vec<super::QuicDecodedBlock>,
round: u32,
block_counts: Option<&mut BTreeMap<(u32, u8), (u64, u64)>>,
drain_mode: super::NativeSymbolDrainMode,
max_batches: usize,
) -> Result<(u64, u64), QuicTransportError> {
let drain_started = Instant::now(); // ubs:ignore - monotonic intake timing, not crypto randomness
let (observed, accepted, completed_blocks) = super::drain_native_symbol_datagrams_with_blocks(
cx,
&mut link.conn,
manifest,
decoders,
config,
decode_stats,
round,
block_counts,
drain_mode,
max_batches,
)
.await?;
intake_stats.record_symbol_drain(
drain_started.elapsed(),
observed,
accepted,
completed_blocks.len(),
);
completed_backlog.extend(completed_blocks);
Ok((observed, accepted))
}
async fn join_native_receiver_decode_jobs(
cx: &Cx,
decoders: &mut [super::QuicEntryDecoder],
config: &QuicConfig,
decode_stats: &mut super::QuicDecodeStats,
intake_stats: &mut NativeReceiverIntakeStats,
completed_backlog: &mut Vec<super::QuicDecodedBlock>,
) -> Result<(), QuicTransportError> {
let completed =
super::join_native_symbol_decode_jobs_with_blocks(cx, decoders, config, decode_stats)
.await?;
intake_stats.record_completed_blocks(completed.len());
completed_backlog.extend(completed);
Ok(())
}
async fn pump_native_receiver_ready(
cx: &Cx,
link: &mut QuicLink,
intake_stats: &mut NativeReceiverIntakeStats,
) -> Result<usize, QuicTransportError> {
let pump_started = Instant::now(); // ubs:ignore - monotonic pump timing, not crypto randomness
let pumped_packets = link
.pump_inbound_for_with_drain_budget(
cx,
Duration::ZERO,
RECEIVER_SYMBOL_DRAIN_BATCHES_PER_SOCKET_POLL,
)
.await?;
intake_stats.record_pump(pump_started.elapsed(), pumped_packets);
Ok(pumped_packets)
}
async fn write_completed_native_blocks(
cx: &Cx,
link: &mut QuicLink,
control: &mut NativeQuicFrameTransport,
manifest: &TransferManifest,
staged: &mut [QuicStagedEntryReceive],
config: &QuicConfig,
intake_stats: &mut NativeReceiverIntakeStats,
completed_backlog: &mut Vec<super::QuicDecodedBlock>,
) -> Result<(), QuicTransportError> {
for block in completed_backlog.drain(..) {
send_and_flush_native_keep_alive(cx, link, control).await?;
let entry = manifest
.entries
.iter()
.find(|entry| entry.index == block.entry)
.ok_or_else(|| {
QuicTransportError::Integrity(format!(
"decoded block for unknown entry {}",
block.entry
))
})?;
let staged_entry = staged
.get_mut(usize::try_from(block.entry).unwrap_or(usize::MAX))
.ok_or_else(|| {
QuicTransportError::Integrity(format!(
"decoded block for out-of-range entry {}",
block.entry
))
})?;
let write_started = Instant::now(); // ubs:ignore - monotonic staging timing, not crypto randomness
keep_peer_alive_while(
cx,
link,
control,
staged_entry.write_block(entry, block.sbn, &block.data, config),
)
.await?;
intake_stats.record_staging_write(write_started.elapsed(), block.data.len());
quic_rqtrace!(
"receiver: write_block entry={} sbn={} bytes={} elapsed_ms={}",
block.entry,
block.sbn,
block.data.len(),
write_started.elapsed().as_millis(),
);
trace_blocking_roundtrip_probe("after_write_block").await;
send_and_flush_native_keep_alive(cx, link, control).await?;
}
Ok(())
}
/// Drive a full ATP-over-QUIC receive over an established link: Hello+ack,
/// manifest, then symbol rounds with fountain feedback until every entry decodes,
/// then verify + atomic commit and return a [`ReceiveReport`].
async fn run_receiver_session(
cx: &Cx,
link: &mut QuicLink,
dest_dir: &Path,
config: &QuicConfig,
peer_id: &str,
options: &QuicReceiveOptions,
) -> Result<ReceiveReport, QuicTransportError> {
let mut config = super::effective_quic_receiver_config(config)?;
config.validate()?;
let symbol_auth = config.symbol_auth_context()?;
let symbol_auth_enabled = symbol_auth.is_some();
let mut control = NativeQuicFrameTransport::for_stream(super::first_client_bidi_stream());
// Hello + ack.
let hello_frame = link
.next_control_frame(cx, &mut control, "receive sender handshake")
.await?;
if hello_frame.frame_type() != FrameType::Handshake {
return Err(QuicTransportError::Unexpected {
got: hello_frame.frame_type(),
expected: "Handshake",
});
}
let hello: QuicHello = super::parse_json(&hello_frame)?;
let reason = super::reject_hello_reason(&hello, &config, symbol_auth_enabled);
let accepted = reason.is_none();
let delta_handshake = if accepted
&& config.enable_delta
&& super::quic_delta_control_auth_context(&config).is_some()
{
match hello.delta_transfer_nonce {
Some(sender_nonce) => {
let receiver_nonce =
super::fresh_quic_delta_nonce(cx, b"receiver", Some(sender_nonce))?;
Some(super::QuicDeltaHandshakeContext {
sender_nonce,
receiver_nonce,
destination_root: super::quic_delta_destination_root_commitment(
super::quic_delta_control_auth_context(&config).ok_or_else(|| {
QuicTransportError::Config(
"QUIC delta control requires a strict authentication context"
.to_string(),
)
})?,
receiver_nonce,
dest_dir,
)?,
})
}
None => None,
}
} else {
None
};
let accepted_source_stream = accepted
&& hello.source_stream
&& super::quic_native_source_stream_enabled(hello.total_bytes, &config, &link.conn);
let source_stream_recv_window =
accepted_source_stream.then(quic_source_stream_recv_window_bytes);
let ack = QuicHelloAck {
accepted,
peer_id: peer_id.to_string(),
source_stream: accepted_source_stream,
source_stream_recv_window,
reason: reason.clone(),
delta_transfer_nonce: delta_handshake.map(|binding| binding.sender_nonce),
delta_receiver_nonce: delta_handshake.map(|binding| binding.receiver_nonce),
delta_destination_root: delta_handshake.map(|binding| binding.destination_root),
};
let ack_frame = super::json_frame(FrameType::HandshakeAck, &ack)?;
control.send(cx, &mut link.conn, &ack_frame)?;
link.flush(cx).await?;
let mut ack_frames = link.last_flushed_stream_frames();
if let Some(reason) = reason {
return Err(QuicTransportError::HandshakeRejected(reason));
}
// Adopt the sender's bounded-accepted (validated in `reject_hello_reason`: aligned,
// >= floor, <= 32 MiB cap) scaled block geometry so the receiver's decoders match the
// sender's encode geometry for large entries. The mod.rs `receive_established_native_connection`
// path already does this (ASUP-E802, 2ff8a0a5d) but the CLI `run_receiver_session` path did
// not — without it `decoders_from_manifest` rejects any source object > max_block_size*256
// (~128 MiB at the 512 KiB default), so encrypted transfers >= ~128 MiB fail closed
// (br-asupersync-j73ili, MATRIX-201/202).
if let Ok(hello_block) = usize::try_from(hello.max_block_size) {
config.max_block_size = hello_block;
}
let config = &config;
let source_stream = if accepted_source_stream {
super::source_stream_from_hello(&hello)?
} else {
None
};
if let (Some(source_stream), Some(window)) = (source_stream, source_stream_recv_window) {
// Install the bounded receive window advertised in the HelloAck before
// any source bytes arrive; the read pump's per-chunk ACK flushes carry
// the MAX_STREAM_DATA advertisements from here on.
link.conn
.configure_stream_recv_window(cx, source_stream, window)?;
// Let the window grow (doubling, up to the fragment-safe cap) when the
// sender reports STREAM_DATA_BLOCKED at our edge after a full window
// was consumed — the sender only reports it while loss-clean.
if let Some(cap) = quic_source_stream_recv_window_growth_cap(
window,
quic_source_stream_recv_window_max_bytes(),
link.max_app_payload,
) {
link.conn.allow_stream_recv_window_growth(
cx,
source_stream,
cap,
crate::net::quic_native::connection::MAX_BUFFERED_STREAM_REASSEMBLY_FRAGMENTS,
)?;
}
link.flush(cx).await?;
}
// Manifest.
let manifest_frame = loop {
let frame = link
.next_control_frame_with_stream_pto(
cx,
&mut control,
"receive transfer manifest",
&ack_frames,
"handshake_ack_pto",
)
.await?;
match frame.frame_type() {
FrameType::ObjectManifest => break frame,
FrameType::Handshake => {
let duplicate: QuicHello = super::parse_json(&frame)?;
if duplicate != hello {
return Err(QuicTransportError::Unexpected {
got: FrameType::Handshake,
expected: "ObjectManifest or duplicate Handshake",
});
}
control.send(cx, &mut link.conn, &ack_frame)?;
link.flush(cx).await?;
ack_frames = link.last_flushed_stream_frames();
}
FrameType::KeepAlive => {}
got => {
return Err(QuicTransportError::Unexpected {
got,
expected: "ObjectManifest",
});
}
}
};
let (manifest, delta_session) = if let Some(handshake) = delta_handshake {
let envelope: super::QuicDeltaManifestEnvelope = super::parse_json_frame(
&manifest_frame,
FrameType::ObjectManifest,
"authenticated QUIC delta ObjectManifest",
)?;
super::validate_quic_manifest(&envelope.manifest, config)?;
let session = super::derive_quic_delta_session(
handshake,
&hello.peer_id,
peer_id,
&envelope.manifest,
)?;
let context = super::quic_delta_control_auth_context(config).ok_or_else(|| {
QuicTransportError::Config(
"QUIC delta control requires a strict authentication context".to_string(),
)
})?;
super::validate_quic_delta_manifest_envelope(context, session, &envelope)?;
(envelope.manifest, Some(session))
} else {
let manifest: TransferManifest =
super::parse_json_frame(&manifest_frame, FrameType::ObjectManifest, "ObjectManifest")?;
super::validate_quic_manifest(&manifest, config)?;
(manifest, None)
};
super::quic_progress(format_args!(
"receiver: manifest transfer={} total_bytes={} entries={} symbol_size={} max_block_size={}",
manifest.transfer_id,
manifest.total_bytes,
manifest.entries.len(),
config.symbol_size,
config.max_block_size
));
link.flush(cx).await?;
let mut delta_full_request_frame = None;
if let Some(session) = delta_session {
let request =
super::build_quic_receiver_delta_request(cx, dest_dir, config, session, &manifest)
.await?;
let request_mode = super::validate_quic_delta_request(&request, session, &manifest)?;
let request_frame = super::json_frame(FrameType::ObjectRequest, &request)?;
control.send(cx, &mut link.conn, &request_frame)?;
link.flush(cx).await?;
let mut request_frames = link.last_flushed_stream_frames();
link.wait_for_stream_frames_acked(
cx,
&mut request_frames,
"confirm bound QUIC delta request delivery",
"delta_request_delivery_pto",
)
.await?;
match request_mode {
super::DeltaWireMode::FullObject => delta_full_request_frame = Some(request_frame),
super::DeltaWireMode::AlreadyInSync => {
let complete = loop {
let frame = link
.next_control_frame_with_stream_pto(
cx,
&mut control,
"receive QUIC delta no-op completion",
&request_frames,
"delta_request_pto",
)
.await?;
match frame.frame_type() {
FrameType::ObjectComplete => {
break super::parse_quic_round_complete(&frame)?;
}
FrameType::ObjectManifest => {
if frame != manifest_frame {
return Err(QuicTransportError::Integrity(
"sender changed the QUIC delta manifest after negotiation"
.to_string(),
));
}
control.send(cx, &mut link.conn, &request_frame)?;
link.flush(cx).await?;
request_frames = link.last_flushed_stream_frames();
}
FrameType::KeepAlive => {}
got => {
return Err(QuicTransportError::Unexpected {
got,
expected: "ObjectComplete | ObjectManifest | KeepAlive",
});
}
}
};
if complete.round != 0 || complete.round_symbols_sent != 0 {
return Err(QuicTransportError::Integrity(format!(
"QUIC delta no-op completion carried round={} symbols={}",
complete.round, complete.round_symbols_sent
)));
}
let revalidated = super::build_quic_receiver_delta_request(
cx, dest_dir, config, session, &manifest,
)
.await?;
if super::validate_quic_delta_request(&revalidated, session, &manifest)?
!= super::DeltaWireMode::AlreadyInSync
{
return Err(QuicTransportError::Integrity(
"destination changed before QUIC delta no-op proof".to_string(),
));
}
let committed_path =
super::quic_safe_base_for_root_name(dest_dir, &manifest.root_name)?;
let receipt = ReceiveReceipt {
committed: true,
bytes_received: 0,
files: super::manifest_logical_files(&manifest),
sha_ok: true,
merkle_ok: true,
symbols_accepted: 0,
feedback_rounds: 0,
decode_count: 0,
decode_micros: 0,
reason: None,
committed_paths: vec![committed_path.display().to_string()],
};
let proof = super::make_quic_delta_proof(session, &manifest, receipt.clone());
send_native_proof_until_close(cx, link, &mut control, &proof, &receipt, config)
.await?;
let _ = super::send_native_close(cx, &mut link.conn, &mut control);
let _ = link.flush(cx).await;
return Ok(ReceiveReport {
transfer_id: manifest.transfer_id.clone(),
bytes_received: 0,
files: receipt.files,
committed: true,
symbols_accepted: 0,
feedback_rounds: 0,
decode_count: 0,
decode_micros: 0,
committed_paths: vec![committed_path],
peer: link.peer,
});
}
super::DeltaWireMode::DeltaChunks => unreachable!(
"build_quic_receiver_delta_request never emits missing-chunk mode in this rollout"
),
}
}
let mut decoders = super::decoders_from_manifest(&manifest, config)?;
super::prepare_quic_destination_root(dest_dir).await?;
let mut staging_guard = None;
for _ in 0..32 {
let staging_seq = QUIC_STAGING_SEQ.fetch_add(1, Ordering::Relaxed);
let staging_nonce = quic_staging_nonce_hex()?;
let candidate = dest_dir.join(format!(
".atp-quic-staging-{}-{staging_nonce}-{staging_seq}",
manifest.transfer_id
));
match create_quic_staging_dir_guard(candidate).await {
Ok(guard) => {
super::prepare_quic_destination_root(dest_dir).await?;
super::reject_quic_existing_symlink(&guard.dir).await?;
staging_guard = Some(guard);
break;
}
Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {}
Err(error) => return Err(error.into()),
}
}
let mut staging_guard = staging_guard.ok_or_else(|| {
QuicTransportError::Source(format!(
"unable to create a unique owned QUIC staging directory under {}",
dest_dir.display()
))
})?;
let staging_dir = staging_guard.dir.clone();
let mut staged = manifest
.entries
.iter()
.enumerate()
.map(|(i, entry)| {
QuicStagedEntryReceive::new_with_options(
staging_dir.join(i.to_string()),
entry.size,
manifest.entries.len(),
options,
)
})
.collect::<Vec<_>>();
if let Some(source_stream) = source_stream {
let previous_paced_source_stream = link.paced_source_stream.replace(source_stream);
link.source_stream_observed_end = 0;
link.source_stream_observed_fin_end = None;
let source_receive_result: Result<ReceiveReport, QuicTransportError> = async {
if hello.total_bytes != manifest.total_bytes {
return Err(QuicTransportError::Integrity(format!(
"source stream hello total_bytes {} did not match manifest total_bytes {}",
hello.total_bytes, manifest.total_bytes
)));
}
if !super::quic_native_source_stream_enabled(manifest.total_bytes, config, &link.conn) {
return Err(QuicTransportError::Integrity(format!(
"sender advertised QUIC source stream {} for an ineligible or oversized transfer",
source_stream.0
)));
}
let (bytes_received, symbols_accepted, decode_stats) =
receive_native_source_stream_entries_pumped(
cx,
link,
source_stream,
&manifest,
&mut staged,
config,
)
.await?;
quic_rqtrace!(
"receiver: native_source_stream received bytes={} stream={} symbols_accepted={} decode_count={}",
bytes_received,
source_stream.0,
symbols_accepted,
decode_stats.decode_count
);
let (mut receipt, committed_paths) = commit_staged_entries(
cx,
link,
&mut control,
dest_dir,
&manifest,
&mut staged,
config,
options,
)
.await?;
receipt.symbols_accepted = symbols_accepted;
receipt.feedback_rounds = 0;
receipt.decode_count = decode_stats.decode_count;
receipt.decode_micros = decode_stats.decode_micros;
send_native_proof_until_close(
cx,
link,
&mut control,
&receipt,
&receipt,
config,
)
.await?;
let _ = super::send_native_close(cx, &mut link.conn, &mut control);
let _ = link.flush(cx).await;
if !receipt.committed {
return Err(QuicTransportError::Integrity(
receipt
.reason
.clone()
.unwrap_or_else(|| "receiver did not commit".to_string()),
));
}
Ok(ReceiveReport {
transfer_id: manifest.transfer_id.clone(),
bytes_received: receipt.bytes_received,
files: receipt.files,
committed: true,
symbols_accepted: receipt.symbols_accepted,
feedback_rounds: receipt.feedback_rounds,
decode_count: receipt.decode_count,
decode_micros: receipt.decode_micros,
committed_paths,
peer: link.peer,
})
}
.await;
link.paced_source_stream = previous_paced_source_stream;
for staged_entry in &mut staged {
let _ = staged_entry.close_cached_staging_file().await;
}
match crate::fs::remove_dir_all(&staging_dir).await {
Ok(()) => staging_guard.disarm(),
Err(error) if error.kind() == std::io::ErrorKind::NotFound => staging_guard.disarm(),
Err(error) if source_receive_result.is_ok() => return Err(error.into()),
Err(_) => {}
}
return source_receive_result;
}
let mut symbols_accepted = 0u64;
let mut feedback_rounds = 0u32;
let mut decode_stats = super::QuicDecodeStats::default();
let mut intake_stats = NativeReceiverIntakeStats::default();
// Control-plane PTO state: the last NeedMore we sent, the STREAM offsets that
// carried it, and how many times we have requeued those offsets while awaiting
// this round's repair. This fills a missing STREAM gap instead of appending a
// duplicate NeedMore behind bytes the peer cannot yet read.
let mut last_need: Option<(QuicNeedMore, Vec<SentControlStreamFrame>)> = None;
let mut needmore_pto_attempts = 0u32;
let needmore_pto_max_attempts = needmore_pto_attempt_budget(config.idle_timeout);
let receive_result: Result<ReceiveReport, QuicTransportError> = async {
'rounds: loop {
cx.checkpoint().map_err(|_| QuicTransportError::Cancelled)?;
let mut round_symbols_observed = 0u64;
let mut round_symbols_accepted = 0u64;
// Drain symbols and wait for this round's ObjectComplete marker, pumping
// the socket and draining the bounded inbound DATAGRAM queue each step so
// no symbol is dropped while we wait. `pump_inbound` returns 0 only after a
// full idle window with no traffic, which means the sender went silent. If
// the round made progress, silence is enough to request repair for the
// remaining gaps even when the best-effort ObjectComplete marker was lost.
let expected_round_complete = feedback_rounds;
let mut round_complete = super::QuicRoundComplete {
round: expected_round_complete,
..super::QuicRoundComplete::default()
};
let mut completed_backlog = Vec::new();
let mut round_block_symbols = BTreeMap::<(u32, u8), (u64, u64)>::new();
loop {
let (observed, accepted) = drain_native_receiver_symbol_queue(
cx,
link,
&manifest,
&mut decoders,
config,
&mut decode_stats,
&mut intake_stats,
&mut completed_backlog,
expected_round_complete,
Some(&mut round_block_symbols),
super::NativeSymbolDrainMode::ReadyOnly,
RECEIVER_SYMBOL_DRAIN_BATCHES_PER_SOCKET_POLL,
)
.await?;
round_symbols_observed = round_symbols_observed.saturating_add(observed);
round_symbols_accepted = round_symbols_accepted.saturating_add(accepted);
symbols_accepted = symbols_accepted.saturating_add(accepted);
if observed > 0 {
// Repair (or spray) is flowing again — reset the control-PTO budget.
needmore_pto_attempts = 0;
send_native_keep_alive(cx, &mut link.conn, &mut control)?;
}
let ready_pumped = pump_native_receiver_ready(cx, link, &mut intake_stats).await?;
quic_rqtrace!(
"receiver: round_step round={} observed={observed} accepted={accepted} ready_pumped={ready_pumped} pending_datagrams={} parked={} pending_decodes={}",
feedback_rounds,
link.conn.pending_datagram_count(),
link.pending_inbound_packets(),
decoders
.iter()
.map(|decoder| decoder.pending_decodes.len())
.sum::<usize>(),
);
if ready_pumped > 0 || link.conn.pending_datagram_count() > 0 {
continue;
}
write_completed_native_blocks(
cx,
link,
&mut control,
&manifest,
&mut staged,
config,
&mut intake_stats,
&mut completed_backlog,
)
.await?;
if super::pending_entries(&decoders).is_empty() {
// Once all entries decode, Proof can complete the transfer even
// if the best-effort ObjectComplete control packet was dropped.
flush_cached_quic_staging_files(&mut staged).await?;
emit_receiver_round_block_symbols(
manifest.transfer_id.as_str(),
expected_round_complete,
&round_block_symbols,
);
break 'rounds;
}
if let Some(frame) = control.try_recv(cx, &mut link.conn)? {
match frame.frame_type() {
FrameType::ObjectComplete => {
let complete = super::parse_quic_round_complete(&frame)?;
if complete.round != expected_round_complete {
trace_stale_round_complete(cx, expected_round_complete, &complete);
continue;
}
super::quic_progress(format_args!(
"receiver: object_complete_received round={} transfer={} round_symbols_sent={} observed={} accepted={}",
complete.round,
manifest.transfer_id,
complete.round_symbols_sent,
round_symbols_observed,
round_symbols_accepted
));
round_complete = complete;
if pump_native_receiver_ready(cx, link, &mut intake_stats).await? > 0
|| link.conn.pending_datagram_count() > 0
{
continue;
}
join_native_receiver_decode_jobs(
cx,
&mut decoders,
config,
&mut decode_stats,
&mut intake_stats,
&mut completed_backlog,
)
.await?;
write_completed_native_blocks(
cx,
link,
&mut control,
&manifest,
&mut staged,
config,
&mut intake_stats,
&mut completed_backlog,
)
.await?;
if complete.round_symbols_sent > round_symbols_observed {
continue;
}
break;
}
FrameType::ObjectManifest => {
let Some(request_frame) = delta_full_request_frame.as_ref() else {
return Err(QuicTransportError::Unexpected {
got: FrameType::ObjectManifest,
expected: "ObjectComplete | KeepAlive",
});
};
if frame != manifest_frame {
return Err(QuicTransportError::Integrity(
"sender changed the QUIC manifest while awaiting full-object data"
.to_string(),
));
}
control.send(cx, &mut link.conn, request_frame)?;
link.flush(cx).await?;
continue;
}
FrameType::KeepAlive => {
send_and_flush_native_keep_alive(cx, link, &mut control).await?;
continue;
}
got => {
return Err(QuicTransportError::Unexpected {
got,
expected: "ObjectComplete | ObjectManifest | KeepAlive",
});
}
}
}
let round_flushed = link.flush(cx).await?;
let round_made_progress = round_symbols_observed > 0;
let pump_timeout = if round_made_progress {
paced_repair_round_idle_grace(
config,
last_need.as_ref().map(|(need, _)| need),
link.symbol_datagram_frame_len,
)
} else if last_need.is_some() {
// Awaiting a repair round after a NeedMore: poll on the short control-PTO interval
// so a lost NeedMore/repair self-heals quickly rather than stalling for idle_timeout.
NEEDMORE_PTO
} else {
config.idle_timeout
};
let pump_started = Instant::now(); // ubs:ignore - monotonic pump timing, not crypto randomness
let pumped_packets = link
.pump_inbound_for_with_drain_budget(
cx,
pump_timeout,
RECEIVER_SYMBOL_DRAIN_BATCHES_PER_SOCKET_POLL,
)
.await?;
intake_stats.record_pump(pump_started.elapsed(), pumped_packets);
quic_rqtrace!(
"receiver: round_pump round={} timeout_ms={} waited_ms={} flushed={round_flushed} pumped={pumped_packets} pending_datagrams={} parked={} {}",
feedback_rounds,
pump_timeout.as_millis(),
pump_started.elapsed().as_millis(),
link.conn.pending_datagram_count(),
link.pending_inbound_packets(),
link.wait_diagnostics(),
);
if pumped_packets == 0 {
if link.conn.pending_datagram_count() > 0 {
continue;
}
if round_made_progress
&& (last_need.is_none() || round_complete.round_symbols_sent > 0)
{
join_native_receiver_decode_jobs(
cx,
&mut decoders,
config,
&mut decode_stats,
&mut intake_stats,
&mut completed_backlog,
)
.await?;
write_completed_native_blocks(
cx,
link,
&mut control,
&manifest,
&mut staged,
config,
&mut intake_stats,
&mut completed_backlog,
)
.await?;
break;
}
// Idle with no progress. If we are awaiting a repair round, the NeedMore (or the
// repair) was lost on the wire. Requeue the same STREAM offsets instead of
// appending a fresh NeedMore each PTO: appending duplicates can leave an older lost
// duplicate ahead of the newer request and head-of-line block the sender forever.
if let Some((need, need_frames)) = last_need.as_ref() {
if needmore_pto_attempts < needmore_pto_max_attempts {
needmore_pto_attempts = needmore_pto_attempts.saturating_add(1);
let need = need.clone();
let need_frames = need_frames.clone();
quic_rqtrace!(
"receiver: NeedMore PTO retransmit round={} attempt={} pending={} repair_blocks={} requested_repair_symbols={} stream_frames={} max_attempts={}",
feedback_rounds,
needmore_pto_attempts,
need.pending.len(),
need.repair_blocks.len(),
need_more_repair_symbol_count(&need),
need_frames.len(),
needmore_pto_max_attempts,
);
super::quic_progress(format_args!(
"receiver: need_more_pto round={} attempt={} pending={} repair_blocks={} requested_repair_symbols={} stream_frames={} max_attempts={}",
feedback_rounds,
needmore_pto_attempts,
need.pending.len(),
need.repair_blocks.len(),
need_more_repair_symbol_count(&need),
need_frames.len(),
needmore_pto_max_attempts
));
match need_more_pto_mode(&need_frames) {
NeedMorePtoMode::SendFresh => {
super::send_native_need_more(
cx,
&mut link.conn,
&mut control,
&need,
)?;
link.flush(cx).await?;
if let Some((_, stored_need_frames)) = last_need.as_mut() {
*stored_need_frames = link.last_flushed_stream_frames();
}
}
NeedMorePtoMode::RetransmitRecorded => {
link.retransmit_stream_frames(cx, &need_frames, "need_more_pto")
.await?;
}
}
continue;
}
}
return Err(link.symbol_round_timeout(config.idle_timeout, symbols_accepted));
}
}
emit_receiver_round_block_symbols(
manifest.transfer_id.as_str(),
expected_round_complete,
&round_block_symbols,
);
flush_cached_quic_staging_files(&mut staged).await?;
if round_complete.round_symbols_sent == 0 && round_symbols_observed > 0 {
let inferred_symbols = infer_missing_round_complete_symbols(
expected_round_complete,
round_symbols_observed,
last_need.as_ref().map(|(need, _)| need),
);
if inferred_symbols > 0 {
trace_inferred_round_complete_symbols(
cx,
expected_round_complete,
round_symbols_observed,
inferred_symbols,
);
round_complete.round_symbols_sent = inferred_symbols;
}
}
let pending = super::pending_entries(&decoders);
if pending.is_empty() {
break;
}
let next_feedback_round = next_feedback_round_or_no_convergence(
feedback_rounds,
config.max_feedback_rounds,
pending.len(),
)?;
// Request fountain-robust FRESH repair (not fragile specific-source re-send). RaptorQ is
// a fountain code: any K independent symbols decode a block, so fresh repair symbols
// (new ESIs, generated via the sender's per-block repair cursor) fill a block's deficit
// regardless of WHICH specific source symbols were lost — and they are always valid.
// The old specific-source path (`source_symbols`) over-reported missing symbols (it
// recomputes the deficit before the paced datagrams the reliable ObjectComplete raced
// ahead of have settled) AND could request `esi >= block_k` for the last partial block,
// which made the sender's `native_source_symbol_for_request` error out and die, so the
// receiver idled to a timeout. `repair_blocks` routes the sender to fresh repair for
// the incomplete source blocks only, so a single final block is not starved by complete
// blocks in the same pending entry.
let round_loss_fraction = super::receiver_round_loss_fraction(
round_symbols_observed,
round_complete.round_symbols_sent,
);
let repair_symbol_round_cap =
super::quic_repair_symbol_round_cap(config, round_loss_fraction);
let (repair_blocks, repair_accounting) = super::block_repair_requests_with_accounting(
&decoders,
config,
repair_symbol_round_cap,
round_loss_fraction,
next_feedback_round,
);
let progress = super::quic_pending_decode_progress(&decoders, &pending, config);
let need = QuicNeedMore {
feedback_round: next_feedback_round,
pending,
repair_blocks,
source_symbols: Vec::new(),
round_symbols_observed: Some(round_symbols_observed),
round_loss_fraction,
round_symbols_accepted: Some(round_symbols_accepted),
repair_base_deficit_symbols: Some(repair_accounting.base_deficit_symbols),
repair_loss_compensated_target_symbols: Some(
repair_accounting.loss_compensated_target_symbols,
),
repair_request_gap_to_target_symbols: Some(
repair_accounting.request_gap_to_target_symbols,
),
repair_symbol_round_cap: Some(
u64::try_from(repair_symbol_round_cap).unwrap_or(u64::MAX),
),
pending_rank: Some(progress.rank),
pending_rank_columns: Some(progress.rank_columns),
pending_rank_deficit: Some(progress.rank_deficit),
pending_decode_jobs: Some(progress.pending_decode_jobs),
};
intake_stats.trace_need_more(cx, next_feedback_round, &need);
let requested_repair_symbols = need_more_repair_symbol_count(&need);
let base_deficit_symbols =
need_more_base_deficit_symbols(&need, requested_repair_symbols);
let loss_compensated_target_symbols =
need_more_loss_compensated_target_symbols(&need, base_deficit_symbols);
let request_gap_to_target = need
.repair_request_gap_to_target_symbols
.unwrap_or_else(|| {
loss_compensated_target_symbols.saturating_sub(requested_repair_symbols)
});
trace_native_repair_accounting(
cx,
"receiver",
next_feedback_round,
base_deficit_symbols,
requested_repair_symbols,
loss_compensated_target_symbols,
None,
&need,
);
let repair_detail = repair_block_trace_summary(&need.repair_blocks);
quic_rqtrace!(
"receiver: NeedMore round={} pending={} repair_blocks={} base_deficit_symbols={} requested_repair_symbols={} loss_compensated_target_symbols={} request_gap_to_target={} source_requests={} round_symbols_observed={} round_symbols_accepted={} round_symbols_sent={} round_loss_fraction={:.4} symbols_accepted={} max_feedback_rounds={} round_cap_exceeded={} repair_symbol_round_cap={} repair_blocks_detail={}",
next_feedback_round,
need.pending.len(),
need.repair_blocks.len(),
base_deficit_symbols,
requested_repair_symbols,
loss_compensated_target_symbols,
request_gap_to_target,
need.source_symbols.len(),
round_symbols_observed,
round_symbols_accepted,
round_complete.round_symbols_sent,
need.round_loss_fraction.unwrap_or(0.0),
symbols_accepted,
config.max_feedback_rounds,
next_feedback_round > config.max_feedback_rounds,
repair_symbol_round_cap,
repair_detail,
);
super::quic_progress(format_args!(
"receiver: need_more_sent round={} transfer={} pending={} repair_blocks={} requested_repair_symbols={} source_requests={} observed={} accepted={} round_symbols_sent={} round_loss_fraction={:.4} repair_blocks_detail={}",
next_feedback_round,
manifest.transfer_id,
need.pending.len(),
need.repair_blocks.len(),
requested_repair_symbols,
need.source_symbols.len(),
round_symbols_observed,
round_symbols_accepted,
round_complete.round_symbols_sent,
need.round_loss_fraction.unwrap_or(0.0),
repair_detail
));
trace_repair_block_deficits(
"receiver",
next_feedback_round,
&need.repair_blocks,
);
super::send_native_need_more(cx, &mut link.conn, &mut control, &need)?;
link.flush(cx).await?;
let need_frames = link.last_flushed_stream_frames();
super::quic_progress(format_args!(
"receiver: need_more_flushed round={} transfer={} stream_frames={}",
next_feedback_round,
manifest.transfer_id,
need_frames.len()
));
// Remember it so the inner loop can re-send it on the control PTO if the repair round
// does not arrive (lost NeedMore/repair); reset the per-round PTO budget.
last_need = Some((need, need_frames));
needmore_pto_attempts = 0;
feedback_rounds = next_feedback_round;
}
NativeReceiveTraceCounters::capture(link).trace_decoded(
cx,
manifest.transfer_id.as_str(),
symbols_accepted,
feedback_rounds,
&decode_stats,
);
intake_stats.trace_summary(cx, manifest.transfer_id.as_str());
send_native_keep_alive(cx, &mut link.conn, &mut control)?;
link.flush(cx).await?;
let (mut receipt, committed_paths) = commit_staged_entries(
cx,
link,
&mut control,
dest_dir,
&manifest,
&mut staged,
config,
options,
)
.await?;
receipt.symbols_accepted = symbols_accepted;
receipt.feedback_rounds = feedback_rounds;
receipt.decode_count = decode_stats.decode_count;
receipt.decode_micros = decode_stats.decode_micros;
send_native_proof_until_close(
cx,
link,
&mut control,
&receipt,
&receipt,
config,
)
.await?;
let _ = super::send_native_close(cx, &mut link.conn, &mut control);
let _ = link.flush(cx).await;
if !receipt.committed {
return Err(QuicTransportError::Integrity(
receipt
.reason
.clone()
.unwrap_or_else(|| "receiver did not commit".to_string()),
));
}
Ok(ReceiveReport {
transfer_id: manifest.transfer_id.clone(),
bytes_received: receipt.bytes_received,
files: receipt.files,
committed: true,
symbols_accepted: receipt.symbols_accepted,
feedback_rounds: receipt.feedback_rounds,
decode_count: receipt.decode_count,
decode_micros: receipt.decode_micros,
committed_paths,
peer: link.peer,
})
}
.await;
for staged_entry in &mut staged {
let _ = staged_entry.close_cached_staging_file().await;
}
// Reclaim the staging directory on every exit path. A successful commit
// renames each entry out (leaving an empty dir); a failed transfer leaves
// orphaned partial blocks behind. Either way the receiver must not leak a
// `.atp-quic-staging-*` directory into the destination — restoring the
// cleanup that 2a3400567 dropped, caught by
// atp_quic_real_udp_transfer_e2e::assert_no_staging_residue.
match crate::fs::remove_dir_all(&staging_dir).await {
Ok(()) => staging_guard.disarm(),
Err(error) if error.kind() == std::io::ErrorKind::NotFound => staging_guard.disarm(),
Err(error) if receive_result.is_ok() => return Err(error.into()),
Err(_) => {}
}
receive_result
}
// ─── Public entry points (called by mod.rs) ─────────────────────────────────
/// Connect to `addr` over real QUIC and transfer the already-prepared source.
/// Drives the full B2 sender coroutine over a real UDP socket.
///
/// `pub(crate)`: it consumes the crate-private prepared-source type and is only
/// reached through the public [`super::send_path`].
pub(crate) async fn send_prepared_over_udp(
cx: &Cx,
addr: SocketAddr,
prepared: &QuicPreparedSource,
config: &QuicConfig,
peer_id: &str,
) -> Result<(SendReport, QuicSendLimiterReport), QuicTransportError> {
let config = prepared.effective_config(config);
config.validate()?;
let client_tls = config.client_tls.as_ref().ok_or_else(|| {
QuicTransportError::Config(
"ATP-over-QUIC send requires client TLS trust config; set QuicConfig::client_tls \
(server name + root certificates) so the server identity can be verified"
.to_string(),
)
})?;
// The session bodies are boxed behind `dyn Future + Send` (one allocation
// per transfer) so a caller that spawns `send_path` / `receive_path` keeps
// its `Send` proof within rustc's default recursion depth
// (br-asupersync-lf8muy); the per-round spray/feedback awaits inside the
// sessions are erased the same way.
let mut link = erase_send(connect(cx, addr, client_tls, &config)).await?;
let report = erase_send(run_sender_session(
cx, &mut link, prepared, &config, peer_id,
))
.await?;
let limiter = link.limiter_report();
Ok((report, limiter))
}
/// Accept one transfer on the pre-bound server `endpoint`, write it under
/// `dest_dir`, verify it, and return a report. Drives the full B3 receiver
/// coroutine over a real UDP socket.
pub async fn receive_on_endpoint(
cx: &Cx,
endpoint: QuicUdpEndpoint,
dest_dir: &Path,
config: &QuicConfig,
peer_id: &str,
) -> Result<ReceiveReport, QuicTransportError> {
receive_on_endpoint_with_options(
cx,
endpoint,
dest_dir,
config,
peer_id,
QuicReceiveOptions::default(),
)
.await
}
/// [`receive_on_endpoint`] with explicit receiver-only filesystem options.
pub async fn receive_on_endpoint_with_options(
cx: &Cx,
endpoint: QuicUdpEndpoint,
dest_dir: &Path,
config: &QuicConfig,
peer_id: &str,
options: QuicReceiveOptions,
) -> Result<ReceiveReport, QuicTransportError> {
options.validate()?;
let config = super::effective_quic_receiver_config(config)?;
let config = &config;
config.validate()?;
let server_tls = config.server_tls.as_ref().ok_or_else(|| {
QuicTransportError::Config(
"ATP-over-QUIC receive requires server TLS config; set QuicConfig::server_tls \
(certificate chain + private key)"
.to_string(),
)
})?;
let (mut link, early_data) = accept(cx, endpoint, server_tls, config).await?;
// Replay any 1-RTT packets that raced ahead of the handshake's completion
// (the client finishes first and may start the data plane immediately), so
// the receiver session sees the sender's Hello / early symbols. Keep them in
// the same bounded replay path as freshly received UDP packets instead of
// bulk-ingesting before manifest parsing creates decoders.
link.queue_received_packets(early_data);
erase_send(run_receiver_session(
cx, &mut link, dest_dir, config, peer_id, &options,
))
.await
}
/// Bind a server UDP endpoint on `listen` for the native QUIC receive path.
pub async fn bind_server_endpoint(
cx: &Cx,
listen: SocketAddr,
) -> Result<QuicUdpEndpoint, QuicTransportError> {
bind_endpoint(cx, listen).await
}
/// GH#67 follow-up (asupersync-t64ggs): a slow-but-alive receiver must not
/// look dead to the sender, and a receiver busy with local work must keep
/// proving it is alive. Mechanism tests over real loopback UDP.
#[cfg(test)]
mod gh67_liveness_tests {
use super::*;
use crate::net::quic_native::handshake_driver::tests::{
CA_CERT_PEM, LEAF_CERT_PEM, leaf_key, parse_one_cert,
};
use crate::net::quic_native::handshake_driver::{client_config, server_config};
use futures_lite::future::{block_on, zip};
async fn established_loopback_links(cx: &Cx, config: &QuicConfig) -> (QuicLink, QuicLink) {
let endpoint = bind_endpoint(cx, "127.0.0.1:0".parse().unwrap())
.await
.expect("bind endpoint");
let address = endpoint.local_addr();
let client_tls = QuicClientTls {
server_name: ServerName::try_from("localhost").unwrap(),
config: client_config(
vec![parse_one_cert(CA_CERT_PEM)],
vec![ATP_QUIC_ALPN.to_vec()],
)
.unwrap(),
};
let server_tls = QuicServerTls {
config: server_config(
vec![parse_one_cert(LEAF_CERT_PEM)],
leaf_key(),
vec![ATP_QUIC_ALPN.to_vec()],
)
.unwrap(),
};
let (client, server) = zip(
connect(cx, address, &client_tls, config),
accept(cx, endpoint, &server_tls, config),
)
.await;
let client = client.expect("client link");
let (mut server, early) = server.expect("server link");
server.ingest_packets(cx, early).expect("early packets");
(client, server)
}
#[test]
fn stream_pto_budget_counts_consecutive_silent_windows() {
assert_eq!(stream_pto_attempts_after_pump(0, 0), 1);
assert_eq!(stream_pto_attempts_after_pump(5, 0), 6);
assert_eq!(
stream_pto_attempts_after_pump(5, 1),
0,
"any 1-RTT packet from the peer restarts the silent-window count"
);
assert_eq!(stream_pto_attempts_after_pump(u32::MAX, 0), u32::MAX);
}
#[test]
fn local_work_keep_alive_fires_once_per_cadence() {
let start = Instant::now();
let cadence = Duration::from_millis(200);
let mut keep_alive = LocalWorkKeepAlive::new(start);
assert!(!keep_alive.due(start, cadence));
assert!(!keep_alive.due(start + Duration::from_millis(199), cadence));
assert!(keep_alive.due(start + Duration::from_millis(200), cadence));
assert!(
!keep_alive.due(start + Duration::from_millis(399), cadence),
"re-based on the emission, not on the poll"
);
assert!(keep_alive.due(start + Duration::from_millis(400), cadence));
}
/// Sender side: with a 3 s idle budget (two 1.5 s windows) the proof wait
/// used to give up after ANY three silent windows in total, even with a
/// keep-alive between each pair. The peer here pings every 2.2 s (one
/// silent window per gap, three gaps) and then answers at ~8 s; the
/// cumulative count died in the third gap (~5.9 s).
#[test]
fn proof_wait_survives_a_slow_but_pinging_peer_over_real_udp() {
block_on(async {
let cx = Cx::for_testing();
let config = QuicConfig {
idle_timeout: Duration::from_secs(3),
..QuicConfig::default()
};
let (mut client, mut server) = established_loopback_links(&cx, &config).await;
let control_stream = super::super::first_client_bidi_stream();
// The client opens its control stream (as the sender's Hello does)
// and writes on it so the server can answer on the same stream.
let mut client_control = NativeQuicFrameTransport::open(&cx, &mut client.conn).unwrap();
assert_eq!(client_control.stream, control_stream);
let mut server_control = NativeQuicFrameTransport::for_stream(control_stream);
let opener = Frame::empty(FrameType::KeepAlive).unwrap();
client_control.send(&cx, &mut client.conn, &opener).unwrap();
client.flush(&cx).await.unwrap();
let opened_at = Instant::now();
loop {
server
.pump_inbound_for(&cx, Duration::from_secs(2))
.await
.unwrap();
if server_control
.try_recv(&cx, &mut server.conn)
.unwrap()
.is_some()
{
break;
}
assert!(
opened_at.elapsed() < Duration::from_secs(10),
"control stream never opened"
);
}
let placeholder = [SentControlStreamFrame {
stream: control_stream,
offset: 0,
len: 1,
}];
let client_wait = async {
let started = Instant::now();
let frame = client
.next_control_frame_with_stream_pto(
&cx,
&mut client_control,
"gh67 proof wait",
&placeholder,
"gh67_test_pto",
)
.await;
(started.elapsed(), frame.map(|frame| frame.frame_type()))
};
let server_pinger = async {
for _ in 0..3 {
crate::time::sleep(cx.now(), Duration::from_millis(2200)).await;
send_and_flush_native_keep_alive(&cx, &mut server, &mut server_control)
.await
.unwrap();
}
crate::time::sleep(cx.now(), Duration::from_millis(1400)).await;
let proof = Frame::empty(FrameType::Proof).unwrap();
server_control.send(&cx, &mut server.conn, &proof).unwrap();
server.flush(&cx).await.unwrap();
};
let ((elapsed, frame), ()) = zip(client_wait, server_pinger).await;
assert_eq!(
frame.expect("proof after three keep-alives"),
FrameType::Proof
);
assert!(
elapsed >= Duration::from_millis(7500),
"the wait must have outlived the cumulative budget (~5.9 s): {elapsed:?}"
);
assert!(elapsed < Duration::from_secs(12), "{elapsed:?}");
});
}
/// Receiver side: a long blocking step raced with the keep-alive pinger
/// keeps the peer fed at the cadence (and drains the socket) until the
/// step completes; the step's result is returned unchanged.
#[test]
fn long_blocking_step_keeps_pinging_the_peer_over_real_udp() {
block_on(async {
let cx = Cx::for_testing();
let config = QuicConfig::default();
let (mut client, mut server) = established_loopback_links(&cx, &config).await;
let mut server_control =
NativeQuicFrameTransport::for_stream(super::super::first_client_bidi_stream());
let work = async {
let started = crate::runtime::spawn_blocking_io(|| {
std::thread::sleep(Duration::from_millis(1100));
Ok::<Instant, std::io::Error>(Instant::now())
})
.await
.expect("blocking step");
Ok::<Instant, QuicTransportError>(started)
};
let server_side = async {
let began = Instant::now();
let finished = keep_peer_alive_while(&cx, &mut server, &mut server_control, work)
.await
.expect("raced step result");
(began, finished)
};
let client_listener = async {
let started = Instant::now();
let mut packets = 0usize;
while started.elapsed() < Duration::from_millis(1700) {
packets += client
.pump_inbound_for(&cx, Duration::from_millis(200))
.await
.unwrap();
}
packets
};
let ((began, finished), packets) = zip(server_side, client_listener).await;
assert!(finished.saturating_duration_since(began) >= Duration::from_millis(1000));
assert!(
(4..=8).contains(&packets),
"expected roughly one keep-alive per 200 ms over 1.1 s, got {packets}"
);
});
}
}
#[cfg(test)]
include!("native_link_tests.rs");