//! ATP-over-RaptorQ transport (v1): the *fast, robust* ATP data plane.
//!
//! Where [`crate::net::atp::transport_tcp`] moves bytes over a single reliable
//! TCP stream, this transport is built for saturating the pipe on a lossy,
//! high-latency internet path:
//!
//! - **Data plane = RaptorQ fountain symbols over UDP.** Each file entry is
//! erasure-coded ([`crate::raptorq`], RFC 6330 systematic RaptorQ) into source
//! plus repair symbols. Symbols are *fungible*: any `K (+ε)` of them recover
//! the entry, from any socket, in any order. Loss is absorbed by repair
//! symbols instead of head-of-line-blocking retransmits.
//! - **Multi-socket fan-out.** Symbols are sprayed round-robin across `N` UDP
//! sockets so a single flow's congestion control / per-socket buffer does not
//! cap throughput.
//! - **Reliable control plane = one TCP connection** reusing the canonical
//! `AtpFrameCodec`: handshake (transfer id + receiver UDP port + coding
//! params), the transfer manifest, fountain *NeedMore* feedback, and the final
//! verified receipt.
//!
//! # Integrity (fail-closed, identical guarantee to `transport_tcp`)
//!
//! After decode, the receiver (1) checks every entry's SHA-256 against the
//! manifest and (2) rebuilds the deterministic flat
//! [`crate::atp::object::ObjectGraph`] and compares the flat Merkle root to the
//! manifest root. Only if both hold does it atomically write the destination and
//! report `committed = true`. Any mismatch, oversize entry, unreachable peer, or
//! undecodable transfer is a hard error.
//!
//! # Fountain feedback loop
//!
//! v1 uses a bounded request/response loop rather than a continuous concurrent
//! ARQ, which keeps it correct on the current runtime:
//!
//! 1. Sender sprays every entry's source symbols across the UDP sockets, plus
//! optional initial repair symbols when `repair_overhead > 1.0`, then sends
//! `ObjectComplete` on TCP.
//! 2. Receiver feeds arriving symbols into a per-entry [`DecodingPipeline`].
//! On `ObjectComplete` it replies with either a `Proof` receipt (all entries
//! decoded → verified + committed) or a `NeedMore` list of still-incomplete
//! entry indices.
//! 3. For each `NeedMore` round the sender generates a *fresh* batch of repair
//! symbols (higher ESI range — RaptorQ is rateless) for the listed entries
//! and resprays. Bounded by `max_feedback_rounds`; exhausting them is a hard
//! error, never a silent partial success.
//!
//! On a low-loss path the initial over-provision means round 0 succeeds; the
//! loop only engages under real loss, which the loopback loss-injection test
//! exercises deterministically.
use std::collections::{BTreeMap, BTreeSet};
use std::net::SocketAddr;
use std::path::{Component, Path, PathBuf};
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::{Duration, Instant};
/// Adaptive block-size / overhead / fan-out optimizer; see
/// `docs/atp_rq_adaptive_design.md`.
pub mod adaptive;
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use smallvec::SmallVec;
use crate::bytes::BytesMut;
use crate::codec::Decoder;
use crate::cx::Cx;
use crate::decoding::{
BlockDecodeJob, BlockDecodeKind, BlockDecodeOutcome, DecodingConfig, DecodingPipeline,
DeferredSymbolAcceptResult, MissingSourceSymbol, RejectReason, SymbolAcceptResult,
run_block_decode_job,
};
use crate::encoding::{EncodedSymbol, EncodingPipeline, MAX_SOURCE_BLOCKS, max_object_size};
use crate::io::{AsyncReadExt, AsyncWriteExt, ReadBuf};
use crate::net::atp::bonding::{
BondTransferDescriptor, BondedDonorSymbolEmission, DonorAssignment, schedule_bonded_donor_spray,
};
use crate::net::atp::datagram::beacons::{BeaconMeasurement, BeaconScheduler};
use crate::net::atp::datagram::congestion::{
CongestionConfig, CongestionController, DatagramRateConfig, DatagramRateController,
DatagramRateDecision, DatagramRateSample,
};
use crate::net::atp::loss::detector::{AtpLossDetector, LossRecommendation};
use crate::net::atp::protocol::codec::AtpFrameCodec;
use crate::net::atp::protocol::frames::{Frame, FrameType, MAX_FRAME_SIZE, ProtocolVersion};
use crate::net::atp::protocol::varint::VarInt;
use crate::net::atp::transport_common::{
EntryDigest, MultiObjectSplitConfig, StreamingError, flat_merkle_root_from_digests,
hash_file_streaming, hex_encode, plan_multi_object_split,
};
use crate::net::{TcpListener, TcpStream, UDP_MAX_GSO_SEGMENTS, UdpBufferConfig, UdpSocket};
use crate::security::authenticated::AuthenticatedSymbol;
use crate::security::tag::TAG_SIZE;
use crate::security::{AuthMode, AuthenticationTag, SecurityContext};
use crate::types::resource::{PoolConfig, SymbolPool};
use crate::types::symbol::{ObjectId, ObjectParams, Symbol, SymbolId, SymbolKind};
use adaptive::{AdaptiveController, AdaptivePolicy, BlockPlan, PathEstimate, PathSignalSample};
/// Protocol identifier carried in the handshake; bump on wire-incompatible
/// changes.
pub const ATP_RQ_PROTOCOL: u32 = 3;
/// Magic prefix on every UDP symbol datagram (`"ATRQ"`).
const SYMBOL_MAGIC: u32 = 0x4154_5251;
/// Default RaptorQ symbol payload size.
///
/// Kept small enough that one symbol plus the authenticated datagram header and
/// IPv4/UDP framing stays under a 1500-byte Ethernet MTU, while avoiding the
/// packet-rate tax of 1 KiB symbols on 100 Mbit links.
pub const DEFAULT_SYMBOL_SIZE: u16 = 1400;
/// Default source-block ceiling.
///
/// With 1400-byte symbols this bounds a block at ~5992 source symbols (well
/// under the RFC 6330 K=56403 cap) and lets a single entry span up to 256
/// blocks (SBN is a `u8`), i.e. up to ~2 GiB per encoded object at this default
/// block size. Larger logical files are split into ordered RaptorQ objects by
/// [`split_large_entries`] so each object's K stays bounded (E-12).
pub const DEFAULT_MAX_BLOCK_SIZE: usize = 8 * 1024 * 1024;
/// Target source-symbol count for the effective transfer block size.
///
/// RaptorQ's matrix work grows sharply with K. A K~512 block is small enough to
/// keep decode/repair work bounded on commodity fleet hosts while still sending
/// large enough UDP bursts to amortize control feedback.
const TARGET_SOURCE_SYMBOLS_PER_BLOCK: usize = 512;
/// Byte ceiling for the normal streaming block-size target.
const TARGET_STREAMING_BLOCK_BYTES: usize = 4 * 1024 * 1024;
/// Maximum encoded ATP-RQ symbols sent in one connected UDP batch.
///
/// Match the UDP GSO segment budget so fixed-size RQ symbols fill one
/// super-packet before the sender flushes. Fanout must not multiply this
/// aggregate burst, or a clean round-0 ramp can overrun the receiver despite
/// aggregate pacing.
const RQ_SEND_BATCH_PER_SOCKET: usize = UDP_MAX_GSO_SEGMENTS;
/// Default round-0 repair multiplier.
///
/// The default keeps the fast source-first shape so trusted/lab RQ receivers can
/// repair sparse source-symbol holes directly before falling back to fountain
/// repair. Adaptive per-round FEC can still raise the sprayed repair overhead
/// without changing this receiver-side source-streaming gate.
pub const DEFAULT_REPAIR_OVERHEAD: f64 = 1.0;
/// Default number of UDP sockets the sender sprays across.
///
/// A single stream is the stable default for the clean round-0 pacing ramp.
/// Multi-stream fanout remains opt-in for targeted experiments.
pub const DEFAULT_UDP_FANOUT: usize = 1;
/// Default ceiling on a single transfer's total bytes (receiver buffers + decode
/// matrices live in memory in v1).
pub const DEFAULT_MAX_TRANSFER_BYTES: u64 = 4 * 1024 * 1024 * 1024;
/// Default maximum time a one-shot RQ receiver waits for the control connection.
///
/// This matches the TCP transport's fail-closed accept bound so scripted
/// `atp recv --once` users cannot hang forever when the sender never connects.
pub const DEFAULT_ACCEPT_TIMEOUT: Duration = Duration::from_secs(60);
/// Default maximum time an RQ sender waits to open the TCP control connection.
pub const DEFAULT_CONNECT_TIMEOUT: Duration = Duration::from_secs(60);
/// Maximum number of files a single transfer manifest may declare. This bounds
/// receiver bookkeeping derived from attacker-controlled control-plane JSON.
const MAX_MANIFEST_ENTRIES: usize = 4 * 1024 * 1024;
/// E-15 tree coalescing: files strictly smaller than this become candidates for
/// packing into a combined RaptorQ object. The threshold intentionally includes
/// the benchmark `tree_small` generator's 1 MiB max-size bucket so those entries
/// share the packed-object receiver staging path instead of becoming thousands
/// of cache-skipping single-file RQ objects.
const PACK_THRESHOLD: u64 = 1024 * 1024 + 1;
/// E-15 tree coalescing: target size for a combined RaptorQ object. The packer
/// greedily fills a pack with small files until adding the next would exceed this
/// (a pack always holds at least one file, so a lone file larger than the target
/// but smaller than [`PACK_THRESHOLD`] still forms its own pack). Roughly one
/// RaptorQ object's worth of bytes — large enough to collapse the per-object
/// runtime overhead, small enough that a lost symbol does not span the whole tree.
const PACK_TARGET: u64 = 8 * 1024 * 1024;
/// Keep a staging descriptor hot for large entries only when the manifest is
/// small enough that the receiver cannot retain too many open files.
const ENTRY_STAGING_FILE_CACHE_MIN_BYTES: u64 = 1024 * 1024;
const ENTRY_STAGING_FILE_CACHE_MAX_ENTRIES: usize = 128;
const RQ_SINGLE_FILE_FRAGMENT_STAGING_DIR: &str = ".atp-rq-fragment-staging";
/// Default bound on fountain feedback rounds before failing closed.
pub const DEFAULT_MAX_FEEDBACK_ROUNDS: u32 = 16;
/// Default receiver-side quiet drain after each round-complete marker.
pub const DEFAULT_ROUND_TAIL_DRAIN_MS: u64 = 2;
/// Default source-retransmit feedback rounds.
///
/// Bounded sparse retransmit is default-on for the source-first path because
/// entry-level repair feedback otherwise re-sprays every block of a large file
/// when only a few systematic symbols are missing. After these early rounds the
/// transport falls back to fountain repair for bursty or non-sparse loss.
pub const DEFAULT_SOURCE_RETRANSMIT_ROUNDS: u32 = 2;
/// Hard cap on source-symbol retransmit requests in one feedback frame. Larger
/// loss bursts fall back to fountain repair rather than creating huge JSON
/// control messages.
pub const DEFAULT_MAX_SOURCE_RETRANSMIT_REQUESTS: usize = 8192;
/// Default receiver-side quiet drain after each round-complete marker.
pub const DEFAULT_ROUND_TAIL_DRAIN: Duration = Duration::from_millis(DEFAULT_ROUND_TAIL_DRAIN_MS);
/// Cold-start aggregate sender pace before feedback evidence exists.
///
/// This is deliberately below a typical LAN burst and above the 100 Mbps rsync
/// baseline target. The pacer uses short symbol bursts with sleeps, so the
/// receiver can drain UDP continuously instead of absorbing a full parallel
/// encode burst in the kernel receive buffer.
const RQ_COLD_START_PACING_BPS: u64 = 16 * 1024 * 1024;
const RQ_MIN_PACING_BPS: u64 = 512 * 1024;
const RQ_MAX_PACING_BPS: u64 = 64 * 1024 * 1024;
/// Explicitly loss-free round-0 sprays can probe above cold-start after each
/// cold-start-sized byte step of emitted datagrams. Loss-configured rounds keep
/// the older AIMD floor/cap path; UDP fanout shares one aggregate pacer and one
/// aggregate send batch limit so additional sockets cannot multiply the
/// clean-ramp burst.
const RQ_ROUND0_CLEAN_RAMP_STEP_BYTES: u64 = RQ_COLD_START_PACING_BPS;
const RQ_ROUND0_CLEAN_RAMP_ADD_BYTES_PER_S: u64 = 8 * 1024 * 1024;
const RQ_ROUND0_CLEAN_RAMP_MAX_PACING_BPS: u64 = 128 * 1024 * 1024;
const RQ_ROUND0_CLEAN_RAMP_FANOUT_MAX_PACING_BPS: u64 = 32 * 1024 * 1024;
/// Small, explicitly clean transfers should not pay proactive RaptorQ repair
/// setup in round 0 when the peer does not take the control-source fast lane.
/// Keep this UDP/RQ fallback gate below large-object lanes; large clean should
/// use the disk-backed control-source stream instead.
const RQ_SMALL_CLEAN_SOURCE_ONLY_MAX_BYTES: u64 = 64 * 1024 * 1024;
const RQ_SMALL_CLEAN_SOURCE_ONLY_MAX_REPAIR_OVERHEAD: f64 = 1.001;
/// Maximum ATP wire frame size expressed as a `usize` for control-source sizing.
const RQ_CONTROL_SOURCE_FRAME_MAX_BYTES: usize = MAX_FRAME_SIZE as usize;
/// Maximum no-extension ATP frame header for source `ObjectData` frames:
/// version(1) + ObjectData type(2) + large payload length(4) + extension count(1).
const RQ_CONTROL_SOURCE_WIRE_HEADER_MAX_BYTES: usize = 1 + 2 + 4 + 1;
const RQ_CONTROL_SOURCE_DATA_HEADER: usize = 4 + 8;
const RQ_CONTROL_SOURCE_AUTH_DATA_HEADER: usize = RQ_CONTROL_SOURCE_DATA_HEADER + TAG_SIZE;
const RQ_CONTROL_SOURCE_AUTH_SYMBOL_DOMAIN: &[u8] =
b"asupersync.atp.rq.control-source-data-auth.v1\0";
/// Control-stream payload chunk for the clean/near-clean source-stream lane.
///
/// Fill each ATP frame to the largest safe no-extension `ObjectData` frame.
/// A 500 MiB clean transfer becomes about 500 bounded control frames instead
/// of ~1000 half-full frames or hundreds of thousands of RQ source datagrams.
const RQ_CONTROL_SOURCE_CHUNK_BYTES: usize = RQ_CONTROL_SOURCE_FRAME_MAX_BYTES
- RQ_CONTROL_SOURCE_WIRE_HEADER_MAX_BYTES
- RQ_CONTROL_SOURCE_DATA_HEADER;
const RQ_CONTROL_SOURCE_AUTH_CHUNK_BYTES: usize = RQ_CONTROL_SOURCE_FRAME_MAX_BYTES
- RQ_CONTROL_SOURCE_WIRE_HEADER_MAX_BYTES
- RQ_CONTROL_SOURCE_AUTH_DATA_HEADER;
/// Flush cadence for bulk `ObjectData` frames on the reliable control stream.
///
/// Handshake, feedback, proof, and close frames still flush immediately through
/// `FrameTransport::send`; only the clean source-stream bulk loop batches these
/// writes. The threshold is deliberately above a 100 Mbit / 25 ms BDP while
/// remaining tiny relative to the 100 MB RSS gate because bytes are handed to
/// the OS stream, not buffered in user space.
const RQ_CONTROL_SOURCE_FLUSH_BYTES: usize = 8 * 1024 * 1024;
/// Best-effort socket buffer request for the clean source-stream control path.
const RQ_CONTROL_STREAM_SOCKET_BUFFER_BYTES: usize = 16 * 1024 * 1024;
/// The 2% "bad" matrix path is rate-shaped to 50 mbit. Cold-starting it at
/// 16 MiB/s overshoots the pipe before feedback can correct the spray, causing
/// repair rounds to chase self-inflicted drops. Keep this deliberately narrow:
/// clean/high-BDP uses the clean ramp, good/0.1% stays source-first, and
/// broken/10% gets its own narrower 10 mbit-class cap.
const RQ_BAD_LINK_ROUND0_LOSS_MIN: f64 = 0.015;
const RQ_BAD_LINK_ROUND0_LOSS_MAX: f64 = RQ_MILD_LOSS_PACING_MAX_LOSS;
const RQ_BAD_LINK_ROUND0_PACING_BPS: u64 = 6 * 1024 * 1024;
const RQ_BROKEN_LINK_ROUND0_LOSS_MIN: f64 = RQ_BAD_LINK_ROUND0_LOSS_MAX;
const RQ_BROKEN_LINK_ROUND0_LOSS_MAX: f64 = RQ_SOURCE_FEC_FALLBACK_MAX_OVERHEAD;
const RQ_BROKEN_LINK_ROUND0_PACING_BPS: u64 = 1152 * 1024;
const RQ_COLD_START_BURST_SYMBOLS: usize = 16;
const RQ_ADAPTIVE_BURST_SYMBOLS: usize = 32;
const RQ_RECEIVER_FLOW_CONTROL_WINDOW_MAX_BYTES: u64 = 16 * 1024 * 1024;
const RQ_PACING_MIN_PAUSE: Duration = Duration::from_micros(50);
const RQ_PACING_MAX_PAUSE: Duration = Duration::from_millis(250);
const RQ_ADAPTIVE_MIN_SAMPLES: u32 = 1;
const RQ_ASSUMED_DECODE_SYMBOLS_PER_S: f64 = 250_000.0;
const RQ_CODING_GAMMA: f64 = 1.5;
const RQ_LOSS_EMA_ALPHA: f64 = 0.35;
const RQ_BW_EMA_ALPHA: f64 = 0.35;
const RQ_BW_TROUGH_RECOVERY_ALPHA: f64 = 0.10;
const RQ_LOSS_BAR_MULTIPLIER: f64 = 1.75;
const RQ_PENDING_PRESSURE_LOSS_FLOOR: f64 = 0.05;
const RQ_REGIME_SHIFT_LOSS_DELTA: f64 = 0.20;
/// Keep mild-loss repair rounds from turning sparse feedback into a self-reinforcing crawl.
const RQ_MILD_LOSS_PACING_FLOOR_FRACTION: f64 = 0.50;
const RQ_SOURCE_FIRST_MILD_LOSS_PACING_FLOOR_FRACTION: f64 = 1.0;
const RQ_MILD_LOSS_PACING_MAX_LOSS: f64 = 0.03;
const RQ_STALLED_REPAIR_PRESSURE_MIN: f64 = 0.50;
const RQ_STALLED_REPAIR_PAYLOAD_FRACTION_MAX: f64 = 0.50;
const RQ_SOURCE_FEC_FALLBACK_ALPHA: f64 = 1e-6;
const RQ_SOURCE_FEC_FALLBACK_MAX_OVERHEAD: f64 = 0.50;
const RQ_SOURCE_FEC_FALLBACK_MIN_LOSS_BAR: f64 = 0.01;
const RQ_SOURCE_FEC_FALLBACK_MIN_OVERHEAD: f64 = 0.03;
/// Receiver-observed loss must directly size feedback repair rounds. The adaptive
/// model's smoothed loss bar can be too small after sparse source retransmit
/// rounds; this floor turns a broken-link sample (p≈0.10) into ≈13% fresh repair
/// per block while staying zero for clean/good paths.
const RQ_FEEDBACK_REPAIR_LOSS_ENABLE_MIN: f64 = RQ_ROUND0_TARGET_LOSS_ENABLE_MIN;
const RQ_FEEDBACK_REPAIR_LOSS_MARGIN_FRACTION: f64 = 0.25;
const RQ_FEEDBACK_REPAIR_LOSS_MARGIN_MIN: f64 = 0.005;
const RQ_AIMD_LOSS_DECREASE_THRESHOLD_MIN: f64 = RQ_MILD_LOSS_PACING_MAX_LOSS;
const RQ_AIMD_LOSS_DECREASE_THRESHOLD_MARGIN: f64 = 0.03;
const RQ_AIMD_CLEAN_INCREASE_THRESHOLD: f64 = 0.0015;
const RQ_AIMD_MULTIPLICATIVE_DECREASE: f64 = 0.50;
const RQ_AIMD_ADDITIVE_INCREASE_BYTES_PER_S: u64 = 1024 * 1024;
/// Loss-targeted cells may overrun a slower path during round 0 before any
/// feedback exists. Once the receiver reports real wire loss, use its observed
/// delivery rate as the backoff anchor instead of repeatedly halving below the
/// pipe.
const RQ_LOSS_TARGET_DELIVERY_BACKOFF_HEADROOM: f64 = 1.10;
/// Loss-targeted cells must also react when receiver arrival loss is
/// underreported but confirmed decode/rank progress is far below the offered
/// send rate. Ratios below this are treated as congestion for AIMD/LossDetector.
const RQ_LOSS_TARGET_PROGRESS_STALL_RATIO: f64 = 0.50;
const RQ_LOSS_TARGET_PROGRESS_LOSS_MARGIN: f64 = 0.01;
/// Do not spend proactive round-0 repair on clean and near-clean links. The
/// MATRIX "good" cell is 0.1% loss and must stay on the source-first path.
const RQ_ROUND0_TARGET_LOSS_ENABLE_MIN: f64 = 0.005;
/// Near-clean loss ceiling allowed onto the reliable control-source stream.
///
/// This matches the MATRIX "good" 0.1% loss fixture while keeping bad/lossy
/// regimes on the FEC datagram fountain.
const RQ_CONTROL_SOURCE_STREAM_MAX_LOSS_TARGET: f64 = RQ_ROUND0_TARGET_LOSS_ENABLE_MIN / 5.0;
/// Minimum object size (bytes) for the large-transfer moderate-loss reliable-stream fallback.
/// Below this the FEC datagram spray converges fine on lossy links (e.g. 50M/broken WINS via
/// forward-repair); at/above it the spray rate-collapses on lossy objects, so the reliable
/// control-source stream (TCP retransmit) is used instead. (MATRIX-199 / br-...317hxr.2.5)
const RQ_LARGE_LOSSY_SOURCE_STREAM_MIN_BYTES: u64 = 256 * 1024 * 1024;
/// Moderate loss ceiling for the large-transfer reliable-stream fallback. Covers the MATRIX "bad"
/// 2% fixture but excludes "broken" (10%), where even reliable TCP degrades and the FEC spray is
/// the right tool. Proven: 500M/bad@2% reliable-stream 95.3s beats rsync 97.9s; the FEC spray
/// times out (pacing collapse, 317hxr.2.5). This bound can shrink once the spray collapse is fixed.
const RQ_LARGE_LOSSY_SOURCE_STREAM_MAX_LOSS_TARGET: f64 = 0.03;
/// Convert an explicit path-loss target into a conservative upper bound before
/// feeding the RaptorQ overhead solver. At 2% loss this produces a ~3% sizing
/// input, enough margin for one-round convergence without turning clean links
/// into fixed-overhead transfers.
const RQ_ROUND0_TARGET_LOSS_MARGIN_FRACTION: f64 = 0.25;
const RQ_ROUND0_TARGET_LOSS_MARGIN_MIN: f64 = 0.005;
/// Hard cap on the per-round fractional repair overhead taken from the controller's
/// wire-loss-driven `plan.overhead` (round_tuning). Without this, a round-0 spray that
/// over-paces a slow link self-inflicts high real loss, the wire-loss estimate clamps near
/// 0.9, and plan.overhead explodes (~9.4 ⇒ ~10.7× total), which both crushes the pacing rate
/// (base/(1+overhead)) and sprays ~10× the data. Bounding the fractional overhead at 1.0
/// (≤2× total) covers any realistic wire loss (≤~50%) in one repair round while preventing
/// the pathological blow-up. (MATRIX-12; bead atp-dataplane-redesign-317hxr.2.5.)
const RQ_MAX_ROUND_REPAIR_OVERHEAD: f64 = 1.0;
/// Packets pulled from the UDP socket per receive-pump turn.
///
/// Mirrors the native QUIC inbound pump batch width so RQ drains bursty symbol
/// sprays after one readiness wait instead of waking once per datagram.
const RQ_INBOUND_PUMP_BATCH: usize = 512;
/// Maximum full batches drained after the first ready batch in one pump turn.
const RQ_INBOUND_PUMP_MAX_DRAIN_BATCHES: usize = 64;
/// Minimum authenticated UDP batch worth sending through the blocking pool.
const RQ_AUTH_VERIFY_PARALLEL_MIN_SYMBOLS: usize = 32;
/// Aim for this many HMAC verifications per blocking-pool task.
const RQ_AUTH_VERIFY_TARGET_CHUNK_SYMBOLS: usize = 32;
/// Hard ceiling on one entry's queued RQ repair-decode jobs.
///
/// A single large file is split into many independent bounded-K source blocks.
/// Let that one entry fan those block decoders across 64-core matrix workers;
/// the receiver pump remains async and the transfer-wide budget below reserves
/// CPU/memory.
const RQ_MAX_PENDING_DECODE_JOBS_PER_ENTRY: usize = 64;
/// Hard ceiling on one transfer's queued RQ repair-decode jobs.
const RQ_MAX_PENDING_DECODE_JOBS_PER_TRANSFER_HARD: usize = 64;
/// Minimum CPU cores left for the UDP/control receive pump and filesystem work.
const RQ_DECODE_MIN_CORES_RESERVED_FOR_IO: usize = 1;
/// Upper bound on CPU cores held back from RQ decode on large machines.
const RQ_DECODE_MAX_CORES_RESERVED_FOR_IO: usize = 4;
/// Soft memory envelope for queued RQ repair-decode jobs.
///
/// `BlockDecodeJob` owns a cloned symbol set plus matrix-solve workspace. The
/// width gate estimates that footprint from current block geometry and lowers
/// the effective transfer width before queued decoders can blow past the
/// MATRIX-5 receiver RSS target. Keep this bounded, but high enough that the
/// 500M matrix cell can use most of a 64-core receiver after the large-entry
/// SBN block-size planner lowers K.
const RQ_DECODE_JOB_MEMORY_BUDGET_BYTES: usize = 256 * 1024 * 1024;
const RQ_DECODE_JOB_MEMORY_FLOOR_BYTES: usize = 1024 * 1024;
const RQ_DECODE_JOB_SYMBOL_MEMORY_MULTIPLIER: usize = 1;
/// Small transfers decode cheaper than they schedule onto the blocking pool.
///
/// MATRIX-49: 500M/bad needs parallel block decode, but 50M/bad regressed when
/// a handful of cheap block solves were over-dispatched. Keep those small
/// entries sequential while preserving wide fanout for 500M-class geometry.
/// MATRIX-50 tightened the gate to require both a large entry and many blocks:
/// block count alone can reopen fanout for 50M-class shapes.
const RQ_PARALLEL_DECODE_MIN_ENTRY_BYTES: u64 = 128 * 1024 * 1024;
const RQ_PARALLEL_DECODE_MIN_SOURCE_BLOCKS: usize = 128;
/// RQ repair feedback is round/RTT-bound: do not reject symbols for an
/// undecoded block. Decoded blocks are cleared immediately after commit.
const RQ_REPAIR_RECEIVE_SYMBOL_CAP_PER_BLOCK: usize = usize::MAX;
/// Estimate at least this much extra repair headroom beyond K for one RQ receive block.
///
/// This is now a decode-job memory estimate only. The RQ receiver must not
/// reject repair symbols for an undecoded block, because MATRIX-5 showed that
/// retention bounding adds repair rounds and dominates wall time.
const RQ_REPAIR_SYMBOL_RETENTION_MIN_EXTRA: usize = 256;
/// Tiny quiet window used only after a full batch, matching the native QUIC pump.
const RQ_INBOUND_PUMP_DRAIN_GRACE: Duration = Duration::from_millis(1);
/// Coalesce clean/source-streamed staging writes so the receiver does not issue
/// one async file write per UDP symbol on large clean transfers.
const RQ_SOURCE_STAGE_BUFFER_BYTES: usize = 256 * 1024;
/// Process-unique suffix for RQ receive staging directories.
static RQ_STAGING_SEQ: AtomicU64 = AtomicU64::new(1);
/// Process-unique nonce folded into source-stream transfer ids that are no
/// longer content-merkle-derived.
static RQ_TRANSFER_SEQ: AtomicU64 = AtomicU64::new(1);
const RQ_STAGING_CREATE_ATTEMPTS: u64 = 1024;
/// UDP datagram header size (magic + transfer tag + entry + sbn + esi + kind +
/// len), big-endian.
const DGRAM_HEADER: usize = 4 + 8 + 4 + 1 + 4 + 1 + 2;
/// UDP datagram header plus the authenticated-symbol tag.
const AUTH_DGRAM_HEADER: usize = DGRAM_HEADER + TAG_SIZE;
/// Opt-in receiver staging-write cursor audit (c54to7 diagnosis). When
/// `ATP_RQ_STAGING_CURSOR_AUDIT` is set, the cached-staging-handle write path
/// verifies the skip-seek invariant with a `stream_position` call before every
/// skip-eligible write, logs any desync, and self-heals by re-seeking. Off by
/// default: it costs one extra lseek per skip-eligible write.
fn staging_cursor_audit_enabled() -> bool {
static ENABLED: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
*ENABLED.get_or_init(|| std::env::var_os("ATP_RQ_STAGING_CURSOR_AUDIT").is_some())
}
/// Opt-in stderr tracing for transport bring-up/diagnosis. Off unless the
/// `ATP_RQ_TRACE` env var is set, so the production path stays silent.
macro_rules! rqtrace {
($($arg:tt)*) => {
if std::env::var_os("ATP_RQ_TRACE").is_some() {
eprintln!("[atp-rq] {}", format!($($arg)*));
}
};
}
/// Opt-in stderr tracing for channel-bonded donor spray bring-up.
/// Off unless `ATP_BOND_TRACE` is set.
macro_rules! bondtrace {
($($arg:tt)*) => {
if std::env::var_os("ATP_BOND_TRACE").is_some() {
eprintln!("[ATP_BOND_TRACE] [atp-rq] {}", format!($($arg)*));
}
};
}
/// Transport tuning knobs.
#[derive(Debug, Clone)]
pub struct RqConfig {
/// RaptorQ symbol payload size in bytes.
pub symbol_size: u16,
/// Maximum source-block size in bytes.
pub max_block_size: usize,
/// Extra repair fraction sprayed in round 0 (>= 1.0).
pub repair_overhead: f64,
/// Explicit expected round-0 wire loss as a fraction in `[0, 1)`.
///
/// This is optional and defaults to `0.0`. Matrix benchmark callers can set it
/// from the known netem loss for lossy cells so the first spray includes a
/// calibrated fountain repair budget. Clean and near-clean values stay below
/// `RQ_ROUND0_TARGET_LOSS_ENABLE_MIN` and therefore preserve the source-first
/// path.
pub round0_loss_target: f64,
/// Number of UDP sockets the sender sprays across.
pub udp_fanout: usize,
/// Maximum total bytes a single transfer may carry.
pub max_transfer_bytes: u64,
/// Maximum time a one-shot receiver waits for the TCP control accept.
pub accept_timeout: Duration,
/// Maximum fountain feedback rounds before failing closed.
pub max_feedback_rounds: u32,
/// Receiver-side quiet window after each `ObjectComplete` frame.
///
/// TCP can deliver the control-plane round marker before the receiver has
/// drained all UDP symbols already queued in the kernel. This window lets
/// the receiver consume that tail before it asks for repair symbols.
pub round_tail_drain: Duration,
/// Number of early feedback rounds that may request missing systematic
/// source symbols instead of repair symbols when `repair_overhead <= 1.0`.
///
/// Defaults to zero for WAN throughput. Positive values are intended for
/// controlled lab or very low-loss links where sparse source retransmit is
/// known to converge faster than constructing repair symbols.
pub source_retransmit_rounds: u32,
/// Maximum source-symbol retransmit requests in one feedback frame.
///
/// `0` means unbounded, but only after `source_retransmit_rounds` explicitly
/// opts the transport into source retransmit feedback.
pub max_source_retransmit_requests: usize,
/// Test-only: deterministically drop 1-in-N sprayed source symbols on the
/// sender to exercise the repair/feedback path. 0 disables.
pub debug_drop_one_in: u32,
/// Optional per-symbol authentication context for UDP RaptorQ datagrams and
/// clean-link control-source `ObjectData` frames.
///
/// When present, senders append a tag for each symbol and receivers verify
/// every symbol before decoding. Control-source data frames are also tagged
/// and fail closed before staging writes. The TCP handshake, manifest, and
/// feedback frames still need their own authenticated transport to claim full
/// anti-forgery for the whole control plane.
pub symbol_auth_context: Option<SecurityContext>,
/// Explicit escape hatch for loopback/lab callers that run over a trusted
/// transport and accept integrity-vs-manifest only.
pub allow_unauthenticated_symbols: bool,
}
/// Public per-symbol authentication posture for ATP-over-RaptorQ.
///
/// This reports whether the UDP symbol plane is configured to verify tags. It
/// does not claim full Byzantine symbol-injection protection by itself because
/// the TCP control channel and manifest still need authenticated transport.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum RqSymbolAuthMode {
/// Symbols are signed and verified with a configured [`SecurityContext`].
Authenticated,
/// Symbols are deliberately unauthenticated on a trusted loopback/lab link.
TrustedUnauthenticated,
/// No auth context was configured and no explicit trusted opt-out was set.
MissingAuthenticationContext,
}
impl Default for RqConfig {
fn default() -> Self {
Self {
symbol_size: DEFAULT_SYMBOL_SIZE,
max_block_size: DEFAULT_MAX_BLOCK_SIZE,
repair_overhead: DEFAULT_REPAIR_OVERHEAD,
round0_loss_target: 0.0,
udp_fanout: DEFAULT_UDP_FANOUT,
max_transfer_bytes: DEFAULT_MAX_TRANSFER_BYTES,
accept_timeout: DEFAULT_ACCEPT_TIMEOUT,
max_feedback_rounds: DEFAULT_MAX_FEEDBACK_ROUNDS,
round_tail_drain: DEFAULT_ROUND_TAIL_DRAIN,
source_retransmit_rounds: DEFAULT_SOURCE_RETRANSMIT_ROUNDS,
max_source_retransmit_requests: DEFAULT_MAX_SOURCE_RETRANSMIT_REQUESTS,
debug_drop_one_in: 0,
symbol_auth_context: None,
allow_unauthenticated_symbols: false,
}
}
}
impl RqConfig {
/// Require per-symbol authentication with this context.
#[must_use]
pub fn with_symbol_auth(mut self, context: SecurityContext) -> Self {
self.symbol_auth_context = Some(context);
self.allow_unauthenticated_symbols = false;
self
}
/// Explicitly allow unauthenticated symbols for trusted loopback/lab links.
#[must_use]
pub fn allow_unauthenticated_for_trusted_transport(mut self) -> Self {
self.symbol_auth_context = None;
self.allow_unauthenticated_symbols = true;
self
}
/// Return the configured per-symbol authentication posture.
#[must_use]
pub fn symbol_auth_mode(&self) -> RqSymbolAuthMode {
if self.symbol_auth_context.is_some() {
return RqSymbolAuthMode::Authenticated;
}
if self.allow_unauthenticated_symbols {
return RqSymbolAuthMode::TrustedUnauthenticated;
}
RqSymbolAuthMode::MissingAuthenticationContext
}
/// Validate that the symbol-auth posture is deliberate.
pub fn validate_symbol_auth_mode(&self) -> Result<(), RqError> {
self.symbol_auth_context().map(|_| ())
}
fn symbol_auth_context(&self) -> Result<Option<SecurityContext>, RqError> {
if let Some(context) = &self.symbol_auth_context {
return Ok(Some(context.clone()));
}
if self.allow_unauthenticated_symbols {
return Ok(None);
}
Err(RqError::Authentication(
"ATP RaptorQ transport requires symbol_auth_context; call \
with_symbol_auth(...) or explicitly opt into \
allow_unauthenticated_for_trusted_transport() for loopback/lab use"
.to_string(),
))
}
}
#[derive(Debug, Clone, Copy)]
struct RqRoundTuning {
repair_overhead: f64,
pacing: RqSprayPacing,
}
#[derive(Debug, Clone, Copy)]
struct RqSprayPacing {
path_rate_bps: u64,
datagram_bytes: u32,
max_burst_size: u32,
rtt: Option<Duration>,
loss_detected: bool,
}
impl RqSprayPacing {
fn cold_start(symbol_size: u16) -> Self {
Self::from_rate(
RQ_COLD_START_PACING_BPS,
symbol_size,
RQ_COLD_START_BURST_SYMBOLS,
None,
false,
)
}
#[allow(clippy::cast_precision_loss, clippy::cast_possible_truncation)]
fn from_rate(
rate_bytes_per_sec: u64,
symbol_size: u16,
burst_symbols: usize,
rtt: Option<Duration>,
loss_detected: bool,
) -> Self {
let pacing_rate_bytes_per_sec =
rate_bytes_per_sec.clamp(RQ_MIN_PACING_BPS, RQ_MAX_PACING_BPS);
let symbol_bytes = u64::from(symbol_size.max(1))
.saturating_add(u64::try_from(AUTH_DGRAM_HEADER).unwrap_or(u64::MAX));
let datagram_bytes = u32::try_from(symbol_bytes).unwrap_or(u32::MAX).max(1);
let path_rate_bps = pacing_rate_bytes_per_sec.saturating_mul(8);
let max_burst_size = u32::try_from(burst_symbols.max(1))
.unwrap_or(u32::MAX)
.max(1);
Self {
path_rate_bps,
datagram_bytes,
max_burst_size,
rtt,
loss_detected,
}
}
fn rate_bytes_per_sec(self) -> u64 {
self.path_rate_bps / 8
}
fn set_rate_bytes_per_sec(&mut self, rate_bytes_per_sec: u64, max_rate_bytes_per_sec: u64) {
let rate = rate_bytes_per_sec.clamp(
RQ_MIN_PACING_BPS,
max_rate_bytes_per_sec.max(RQ_MIN_PACING_BPS),
);
self.path_rate_bps = rate.saturating_mul(8);
}
fn burst_bytes(self) -> u64 {
u64::from(self.datagram_bytes).saturating_mul(u64::from(self.max_burst_size))
}
fn configured_bdp_bytes(self) -> Option<u64> {
self.rtt
.map(|rtt| rate_window_bytes(self.rate_bytes_per_sec(), rtt))
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
struct RqSenderWindowProbe {
sent_symbols: u64,
payload_bytes: u64,
wire_bytes: u64,
send_wall_ms: u64,
control_wait_ms: u64,
configured_rate_bytes_per_sec: u64,
observed_payload_bytes_per_sec: u64,
observed_wire_bytes_per_sec: u64,
configured_bdp_bytes: u64,
configured_control_window_bytes: u64,
observed_payload_window_bytes: u64,
observed_wire_window_bytes: u64,
burst_bytes: u64,
burst_symbols: u32,
datagram_bytes: u32,
}
impl RqSenderWindowProbe {
fn new(
pacing: RqSprayPacing,
sent_symbols: u64,
symbol_size: u16,
send_wall: Duration,
control_wait: Duration,
) -> Self {
let payload_bytes = sent_symbols.saturating_mul(u64::from(symbol_size.max(1)));
let wire_bytes = sent_symbols.saturating_mul(u64::from(pacing.datagram_bytes));
let observed_payload_bytes_per_sec = bytes_per_second_ceil(payload_bytes, send_wall);
let observed_wire_bytes_per_sec = bytes_per_second_ceil(wire_bytes, send_wall);
Self {
sent_symbols,
payload_bytes,
wire_bytes,
send_wall_ms: duration_millis_u64(send_wall),
control_wait_ms: duration_millis_u64(control_wait),
configured_rate_bytes_per_sec: pacing.rate_bytes_per_sec(),
observed_payload_bytes_per_sec,
observed_wire_bytes_per_sec,
configured_bdp_bytes: pacing.configured_bdp_bytes().unwrap_or(0),
configured_control_window_bytes: rate_window_bytes(
pacing.rate_bytes_per_sec(),
control_wait,
),
observed_payload_window_bytes: rate_window_bytes(
observed_payload_bytes_per_sec,
control_wait,
),
observed_wire_window_bytes: rate_window_bytes(
observed_wire_bytes_per_sec,
control_wait,
),
burst_bytes: pacing.burst_bytes(),
burst_symbols: pacing.max_burst_size,
datagram_bytes: pacing.datagram_bytes,
}
}
fn peak_window_bytes(self) -> u64 {
self.configured_bdp_bytes
.max(self.configured_control_window_bytes)
.max(self.observed_payload_window_bytes)
.max(self.observed_wire_window_bytes)
}
}
fn trace_sender_window_probe(
phase: &str,
feedback_round: u32,
probe: RqSenderWindowProbe,
peak_window_bytes: u64,
udp_send_acceleration: UdpSendAccelerationReport,
) {
rqtrace!(
"sender: window_probe phase={} feedback_round={} sent_symbols={} payload_bytes={} wire_bytes={} send_wall_ms={} control_wait_ms={} configured_rate_Bps={} observed_payload_Bps={} observed_wire_Bps={} configured_bdp_bytes={} configured_control_window_bytes={} observed_payload_window_bytes={} observed_wire_window_bytes={} peak_window_bytes={} burst_bytes={} burst_symbols={} datagram_bytes={} udp_flushes={} udp_datagrams={} udp_payload_bytes={}",
phase,
feedback_round,
probe.sent_symbols,
probe.payload_bytes,
probe.wire_bytes,
probe.send_wall_ms,
probe.control_wait_ms,
probe.configured_rate_bytes_per_sec,
probe.observed_payload_bytes_per_sec,
probe.observed_wire_bytes_per_sec,
probe.configured_bdp_bytes,
probe.configured_control_window_bytes,
probe.observed_payload_window_bytes,
probe.observed_wire_window_bytes,
peak_window_bytes,
probe.burst_bytes,
probe.burst_symbols,
probe.datagram_bytes,
udp_send_acceleration.flushes,
udp_send_acceleration.datagrams,
udp_send_acceleration.payload_bytes,
);
}
fn bytes_per_second_ceil(bytes: u64, elapsed: Duration) -> u64 {
let nanos = elapsed.as_nanos().max(1);
let rate = u128::from(bytes)
.saturating_mul(1_000_000_000)
.div_ceil(nanos);
u64::try_from(rate).unwrap_or(u64::MAX)
}
fn rate_window_bytes(bytes_per_sec: u64, rtt: Duration) -> u64 {
let nanos = rtt.as_nanos();
if nanos == 0 || bytes_per_sec == 0 {
return 0;
}
let bytes = u128::from(bytes_per_sec)
.saturating_mul(nanos)
.div_ceil(1_000_000_000);
u64::try_from(bytes).unwrap_or(u64::MAX)
}
fn rq_receiver_flow_credit_bytes(config: &RqConfig, pending_bytes: u64) -> u64 {
let symbol_bytes = u64::from(config.symbol_size.max(1));
let datagram_bytes = symbol_bytes
.saturating_add(u64::try_from(AUTH_DGRAM_HEADER).unwrap_or(u64::MAX))
.max(1);
let min_window = datagram_bytes
.saturating_mul(u64::try_from(RQ_ADAPTIVE_BURST_SYMBOLS).unwrap_or(u64::MAX))
.max(symbol_bytes);
if pending_bytes == 0 {
return RQ_RECEIVER_FLOW_CONTROL_WINDOW_MAX_BYTES.max(min_window);
}
pending_bytes.clamp(min_window, RQ_RECEIVER_FLOW_CONTROL_WINDOW_MAX_BYTES)
}
fn duration_for_rate_window(bytes: u64, bytes_per_sec: u64) -> Duration {
if bytes == 0 || bytes_per_sec == 0 {
return Duration::ZERO;
}
let nanos = u128::from(bytes)
.saturating_mul(1_000_000_000)
.div_ceil(u128::from(bytes_per_sec));
Duration::from_nanos(u64::try_from(nanos).unwrap_or(u64::MAX))
}
fn duration_millis_u64(duration: Duration) -> u64 {
u64::try_from(duration.as_millis()).unwrap_or(u64::MAX)
}
struct RqSprayPacer {
controller: CongestionController,
pacing: RqSprayPacing,
shared_decision: Option<DatagramRateDecision>,
round0_ramp: Option<RqRound0CleanPacingRamp>,
small_clean_burst: Option<RqSmallCleanBurstPacer>,
}
#[derive(Debug, Clone, Copy)]
struct RqRound0CleanPacingRamp {
sent_datagrams: u64,
next_step_bytes: u64,
max_rate_bytes_per_sec: u64,
}
#[derive(Debug, Clone, Copy)]
struct RqSmallCleanBurstPacer {
remaining_in_burst: u32,
next_burst_at: Option<Instant>,
}
impl RqSmallCleanBurstPacer {
fn new() -> Self {
Self {
remaining_in_burst: 0,
next_burst_at: None,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
struct RqRound0CleanPacingRampReport {
sent_datagrams: u64,
sent_bytes: u64,
old_rate_bytes_per_sec: u64,
new_rate_bytes_per_sec: u64,
next_step_bytes: u64,
max_rate_bytes_per_sec: u64,
}
impl RqRound0CleanPacingRamp {
fn new(max_rate_bytes_per_sec: u64) -> Self {
Self {
sent_datagrams: 0,
next_step_bytes: RQ_ROUND0_CLEAN_RAMP_STEP_BYTES,
max_rate_bytes_per_sec: max_rate_bytes_per_sec.clamp(
RQ_COLD_START_PACING_BPS.max(RQ_MIN_PACING_BPS),
RQ_ROUND0_CLEAN_RAMP_MAX_PACING_BPS,
),
}
}
fn observe_datagram(
&mut self,
pacing: &mut RqSprayPacing,
) -> Option<RqRound0CleanPacingRampReport> {
self.sent_datagrams = self.sent_datagrams.saturating_add(1);
let sent_bytes = self
.sent_datagrams
.saturating_mul(u64::from(pacing.datagram_bytes.max(1)));
let old_rate = pacing.rate_bytes_per_sec();
let mut changed = false;
while sent_bytes >= self.next_step_bytes
&& pacing.rate_bytes_per_sec() < self.max_rate_bytes_per_sec
{
let current = pacing.rate_bytes_per_sec();
let next = current
.saturating_add(RQ_ROUND0_CLEAN_RAMP_ADD_BYTES_PER_S)
.clamp(RQ_MIN_PACING_BPS, self.max_rate_bytes_per_sec);
if next == current {
break;
}
pacing.set_rate_bytes_per_sec(next, self.max_rate_bytes_per_sec);
self.next_step_bytes = self
.next_step_bytes
.saturating_add(RQ_ROUND0_CLEAN_RAMP_STEP_BYTES);
changed = true;
}
changed.then_some(RqRound0CleanPacingRampReport {
sent_datagrams: self.sent_datagrams,
sent_bytes,
old_rate_bytes_per_sec: old_rate,
new_rate_bytes_per_sec: pacing.rate_bytes_per_sec(),
next_step_bytes: self.next_step_bytes,
max_rate_bytes_per_sec: self.max_rate_bytes_per_sec,
})
}
}
fn round0_clean_ramp_max_rate(config: &RqConfig) -> u64 {
if config.udp_fanout.max(1) == 1 {
RQ_ROUND0_CLEAN_RAMP_MAX_PACING_BPS
} else {
RQ_ROUND0_CLEAN_RAMP_FANOUT_MAX_PACING_BPS
}
}
fn round0_clean_ramp_enabled(config: &RqConfig, pacing: RqSprayPacing) -> bool {
// MATRIX-76: near-clean "good" links remain source-first, but must not take
// the no-feedback high-BDP ramp. A fixed 128 MiB/s probe overruns 200 Mbit
// paths before the first feedback frame can produce a delivery-rate cap.
let loss_free_target = round0_source_first_loss_target(config)
&& (0.0..=f64::EPSILON).contains(&config.round0_loss_target);
// Clean UDP fallback still ramps when the control-source stream is not
// selected, for example debug-drop or non-clean transfer regimes.
config.debug_drop_one_in == 0
&& config.repair_overhead <= RQ_SMALL_CLEAN_SOURCE_ONLY_MAX_REPAIR_OVERHEAD
&& loss_free_target
&& !pacing.loss_detected
&& pacing.rate_bytes_per_sec() <= RQ_COLD_START_PACING_BPS
}
impl RqSprayPacer {
fn new_round0(pacing: RqSprayPacing, config: &RqConfig, force_clean_ramp: bool) -> Self {
let clean_ramp_enabled = round0_clean_ramp_enabled(config, pacing);
let round0_ramp = clean_ramp_enabled
.then(|| RqRound0CleanPacingRamp::new(round0_clean_ramp_max_rate(config)));
if round0_ramp.is_some() {
rqtrace!(
"sender: round0_clean_pacing_ramp enabled start_rate_Bps={} step_bytes={} max_rate_Bps={} udp_fanout={} datagram_bytes={} burst_symbols={} forced={}",
pacing.rate_bytes_per_sec(),
RQ_ROUND0_CLEAN_RAMP_STEP_BYTES,
round0_clean_ramp_max_rate(config),
config.udp_fanout.max(1),
pacing.datagram_bytes,
pacing.max_burst_size,
force_clean_ramp,
);
}
Self::new_with_round0_ramp(pacing, round0_ramp, force_clean_ramp && clean_ramp_enabled)
}
fn new_with_round0_ramp(
pacing: RqSprayPacing,
round0_ramp: Option<RqRound0CleanPacingRamp>,
small_clean_burst: bool,
) -> Self {
let mut controller = CongestionController::new(CongestionConfig::default());
Self::configure_controller(&mut controller, pacing, None);
Self {
controller,
pacing,
shared_decision: None,
round0_ramp,
small_clean_burst: small_clean_burst.then(RqSmallCleanBurstPacer::new),
}
}
fn configure_controller(
controller: &mut CongestionController,
pacing: RqSprayPacing,
shared_decision: Option<DatagramRateDecision>,
) {
if let Some(decision) = shared_decision {
controller.configure_from_rate_decision(
decision,
pacing.datagram_bytes,
pacing.max_burst_size,
);
} else {
controller.configure_for_path_rate(
pacing.path_rate_bps,
pacing.datagram_bytes,
pacing.max_burst_size,
);
}
controller.update_congestion_feedback(pacing.rtt, pacing.loss_detected);
}
fn configure_with_shared_decision(
&mut self,
pacing: RqSprayPacing,
shared_decision: Option<DatagramRateDecision>,
) {
self.pacing = pacing;
self.shared_decision = shared_decision;
self.round0_ramp = None;
self.small_clean_burst = None;
Self::configure_controller(&mut self.controller, pacing, shared_decision);
}
fn pacing(&self) -> RqSprayPacing {
self.pacing
}
fn observe_datagram_sent(&mut self) {
let Some(ramp) = &mut self.round0_ramp else {
return;
};
if let Some(report) = ramp.observe_datagram(&mut self.pacing) {
Self::configure_controller(&mut self.controller, self.pacing, self.shared_decision);
rqtrace!(
"sender: round0_clean_rate_ramp sent_datagrams={} sent_bytes={} old_rate_Bps={} new_rate_Bps={} path_rate_bps={} next_step_bytes={} max_rate_Bps={}",
report.sent_datagrams,
report.sent_bytes,
report.old_rate_bytes_per_sec,
report.new_rate_bytes_per_sec,
self.pacing.path_rate_bps,
report.next_step_bytes,
report.max_rate_bytes_per_sec,
);
}
}
async fn before_send(&mut self, cx: &Cx) -> Result<(), RqError> {
if self.small_clean_burst.is_some() {
return self.before_small_clean_burst_send(cx).await;
}
loop {
let now = Instant::now();
if self.controller.try_consume_send_budget(now) {
return Ok(());
}
let wait = self
.controller
.time_until_send_budget(now)
.clamp(RQ_PACING_MIN_PAUSE, RQ_PACING_MAX_PAUSE);
crate::time::sleep(cx.now(), wait).await;
cx.checkpoint().map_err(|_| RqError::Cancelled)?;
}
}
async fn before_small_clean_burst_send(&mut self, cx: &Cx) -> Result<(), RqError> {
let pacing = self.pacing;
let burst_symbols = pacing
.max_burst_size
.max(u32::try_from(RQ_SEND_BATCH_PER_SOCKET).unwrap_or(u32::MAX));
let Some(burst) = self.small_clean_burst.as_mut() else {
return Ok(());
};
if burst.remaining_in_burst > 0 {
burst.remaining_in_burst = burst.remaining_in_burst.saturating_sub(1);
return Ok(());
}
while let Some(next_burst_at) = burst.next_burst_at {
let now = Instant::now();
let Some(wait) = next_burst_at.checked_duration_since(now) else {
break;
};
let wait = wait.clamp(RQ_PACING_MIN_PAUSE, RQ_PACING_MAX_PAUSE);
crate::time::sleep(cx.now(), wait).await;
cx.checkpoint().map_err(|_| RqError::Cancelled)?;
}
let burst_bytes =
u64::from(pacing.datagram_bytes.max(1)).saturating_mul(u64::from(burst_symbols.max(1)));
let burst_interval = duration_for_rate_window(burst_bytes, pacing.rate_bytes_per_sec());
burst.next_burst_at = Instant::now().checked_add(burst_interval);
burst.remaining_in_burst = burst_symbols.saturating_sub(1);
Ok(())
}
}
struct RqPendingSendBatch {
by_socket: Vec<Vec<Vec<u8>>>,
queued: usize,
}
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
struct RqSendBatchFlushReport {
socket_flushes: usize,
native_flushes: usize,
native_packets: usize,
gso_flushes: usize,
gso_packets: usize,
fallback_flushes: usize,
fallback_packets: usize,
partial_flushes: usize,
error_flushes: usize,
packets_processed: usize,
bytes_processed: usize,
}
impl RqPendingSendBatch {
fn new(fanout: usize) -> Self {
let fanout = fanout.max(1);
Self {
by_socket: (0..fanout).map(|_| Vec::new()).collect(),
queued: 0,
}
}
fn fanout(&self) -> usize {
self.by_socket.len()
}
fn global_flush_symbols(&self) -> usize {
RQ_SEND_BATCH_PER_SOCKET
}
fn push(&mut self, socket_index: usize, payload: Vec<u8>) {
let index = socket_index % self.fanout();
self.by_socket[index].push(payload);
self.queued += 1;
}
fn should_flush(&self) -> bool {
self.queued >= self.global_flush_symbols()
|| self
.by_socket
.iter()
.any(|payloads| payloads.len() >= RQ_SEND_BATCH_PER_SOCKET)
}
async fn flush(
&mut self,
sockets: &mut [UdpSocket],
symbols_sent: &mut u64,
) -> Result<RqSendBatchFlushReport, RqError> {
debug_assert_eq!(self.by_socket.len(), sockets.len().max(1));
if self.queued == 0 {
return Ok(RqSendBatchFlushReport::default());
}
let symbols_before_flush = *symbols_sent;
let mut flush_report = RqSendBatchFlushReport::default();
for (socket_index, payloads) in self.by_socket.iter_mut().enumerate() {
if payloads.is_empty() {
continue;
}
let socket = sockets.get_mut(socket_index).ok_or_else(|| {
RqError::Io(std::io::Error::new(
std::io::ErrorKind::InvalidInput,
"RQ send batch socket index out of range",
))
})?;
let expected = payloads.len();
let report = {
let payload_refs = payloads
.iter()
.map(Vec::as_slice)
.collect::<SmallVec<[_; RQ_SEND_BATCH_PER_SOCKET]>>();
socket.send_connected_batch(&payload_refs).await?
};
*symbols_sent = symbols_sent
.saturating_add(u64::try_from(report.packets_processed).unwrap_or(u64::MAX));
flush_report.socket_flushes += 1;
flush_report.packets_processed += report.packets_processed;
flush_report.bytes_processed += report.bytes_processed;
flush_report.native_flushes += usize::from(report.native_send_batch_used);
if report.native_send_batch_used {
flush_report.native_packets += report.packets_processed;
}
flush_report.gso_flushes += usize::from(report.gso_send_used);
if report.gso_send_used {
flush_report.gso_packets += report.packets_processed;
}
flush_report.fallback_flushes += usize::from(report.fallback_used);
if report.fallback_used {
flush_report.fallback_packets += report.packets_processed;
}
flush_report.error_flushes += usize::from(report.error.is_some());
if report.packets_processed != expected {
let reason = report.error.unwrap_or_else(|| {
format!(
"partial RQ UDP send batch: sent {} of {expected}",
report.packets_processed
)
});
let partial_flushes = flush_report.partial_flushes.saturating_add(1);
rqtrace!(
"sender: udp_batch partial flushes={} native_flushes={} gso_flushes={} fallback_flushes={} partial_flushes={} packets={} bytes={} symbols_before={} symbols_after={} error={}",
flush_report.socket_flushes,
flush_report.native_flushes,
flush_report.gso_flushes,
flush_report.fallback_flushes,
partial_flushes,
flush_report.packets_processed,
flush_report.bytes_processed,
symbols_before_flush,
*symbols_sent,
reason,
);
return Err(RqError::Io(std::io::Error::new(
std::io::ErrorKind::WriteZero,
reason,
)));
}
payloads.clear();
}
self.queued = 0;
rqtrace!(
"sender: udp_batch flushes={} native_flushes={} gso_flushes={} fallback_flushes={} partial_flushes={} packets={} bytes={} symbols_before={} symbols_after={}",
flush_report.socket_flushes,
flush_report.native_flushes,
flush_report.gso_flushes,
flush_report.fallback_flushes,
flush_report.partial_flushes,
flush_report.packets_processed,
flush_report.bytes_processed,
symbols_before_flush,
*symbols_sent,
);
if send_progress_crossed_yield_boundary(symbols_before_flush, *symbols_sent) {
crate::runtime::yield_now().await;
}
Ok(flush_report)
}
#[cfg(test)]
fn queued_count(&self) -> usize {
self.queued
}
#[cfg(test)]
fn socket_batch_len(&self, socket_index: usize) -> usize {
self.by_socket[socket_index].len()
}
}
fn send_progress_crossed_yield_boundary(before: u64, after: u64) -> bool {
after > before && before / 64 != after / 64
}
struct RqReceiverUdpFanout {
sockets: Vec<UdpSocket>,
next_poll: usize,
recv_payload_pool: Vec<Vec<u8>>,
}
impl RqReceiverUdpFanout {
async fn bind(
bind_ip: std::net::IpAddr,
fanout: usize,
recv_buffer_bytes: usize,
) -> std::io::Result<Self> {
let fanout = fanout.max(1);
let mut sockets = Vec::with_capacity(fanout);
for _ in 0..fanout {
let socket = UdpSocket::bind(SocketAddr::new(bind_ip, 0)).await?;
let _ = socket.tune_buffers(UdpBufferConfig {
recv_buffer_bytes: Some(recv_buffer_bytes),
send_buffer_bytes: None,
});
sockets.push(socket);
}
Ok(Self {
sockets,
next_poll: 0,
recv_payload_pool: Vec::with_capacity(RQ_INBOUND_PUMP_BATCH),
})
}
fn len(&self) -> usize {
self.sockets.len()
}
fn local_ports(&self) -> std::io::Result<Vec<u16>> {
self.sockets
.iter()
.map(|socket| socket.local_addr().map(|addr| addr.port()))
.collect()
}
fn poll_recv_any(
&mut self,
task_cx: &std::task::Context<'_>,
rbuf: &mut [u8],
) -> std::task::Poll<std::io::Result<(usize, usize)>> {
use std::task::Poll;
if self.sockets.is_empty() {
return Poll::Ready(Err(std::io::Error::new(
std::io::ErrorKind::NotConnected,
"RQ receiver UDP fanout has no sockets",
)));
}
let socket_count = self.sockets.len();
for offset in 0..socket_count {
let socket_index = (self.next_poll + offset) % socket_count;
match self.sockets[socket_index].poll_recv(task_cx, rbuf) {
Poll::Ready(Ok(n)) => {
self.next_poll = (socket_index + 1) % socket_count;
return Poll::Ready(Ok((socket_index, n)));
}
Poll::Ready(Err(e)) => return Poll::Ready(Err(e)),
Poll::Pending => {}
}
}
Poll::Pending
}
fn poll_recv_batch_any(
&mut self,
task_cx: &mut std::task::Context<'_>,
max_packets: usize,
packet_size: usize,
) -> std::task::Poll<std::io::Result<(usize, crate::net::UdpRecvBatch)>> {
use std::future::Future;
use std::task::Poll;
if self.sockets.is_empty() {
return Poll::Ready(Err(std::io::Error::new(
std::io::ErrorKind::NotConnected,
"RQ receiver UDP fanout has no sockets",
)));
}
let socket_count = self.sockets.len();
for offset in 0..socket_count {
let socket_index = (self.next_poll + offset) % socket_count;
let poll_result = {
let socket = &mut self.sockets[socket_index];
let recv_payload_pool = &mut self.recv_payload_pool;
let mut recv = std::pin::pin!(socket.recv_batch_from_reusing(
max_packets,
packet_size,
recv_payload_pool
));
Future::poll(recv.as_mut(), task_cx)
};
match poll_result {
Poll::Ready(Ok(batch)) => {
self.next_poll = (socket_index + 1) % socket_count;
return Poll::Ready(Ok((socket_index, batch)));
}
Poll::Ready(Err(e)) => return Poll::Ready(Err(e)),
Poll::Pending => {}
}
}
Poll::Pending
}
async fn recv_batch_from_socket(
&mut self,
socket_index: usize,
max_packets: usize,
packet_size: usize,
) -> std::io::Result<crate::net::UdpRecvBatch> {
let socket = self.sockets.get_mut(socket_index).ok_or_else(|| {
std::io::Error::new(
std::io::ErrorKind::InvalidInput,
"RQ receiver UDP fanout socket index out of range",
)
})?;
socket
.recv_batch_from_reusing(max_packets, packet_size, &mut self.recv_payload_pool)
.await
}
fn recycle_recv_batch(&mut self, batch: &mut crate::net::UdpRecvBatch, max_spare: usize) {
batch.recycle_payloads_into(&mut self.recv_payload_pool, max_spare);
}
}
struct RqAdaptiveSendState {
controller: AdaptiveController,
shared_rate: DatagramRateController,
shared_rate_decision: Option<DatagramRateDecision>,
shared_rate_clock_micros: u64,
loss_detector: AtpLossDetector,
beacons: BeaconScheduler,
est: PathEstimate,
symbol_size: u16,
aimd_rate_bps: u64,
aimd_feedback_seen: bool,
last_round_loss_fraction: f64,
loss_ema: f64,
pacing_loss_ema: f64,
pacing_loss_bar: f64,
loss_bar: f64,
bw_ema_bps: f64,
bw_trough_bps: f64,
loss_pacing_cap_bps: Option<u64>,
loss_fec_floor: f64,
regime_shift: bool,
last_pending_rank: Option<u64>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum RqDeliverySampleKind {
InitialOrRepair,
SourceRetransmit,
}
#[derive(Debug, Clone, Copy, Default)]
struct RqNeedMoreProgress {
pending_rank: Option<u64>,
pending_rank_columns: Option<u64>,
pending_rank_deficit: Option<u64>,
pending_decode_jobs: Option<u64>,
}
fn rq_shared_rate_config(config: &RqConfig) -> DatagramRateConfig {
let initial = round0_bad_link_pacing_bps(config).unwrap_or(RQ_COLD_START_PACING_BPS);
let symbol_bytes = u64::from(config.symbol_size.max(1));
DatagramRateConfig {
initial_pacing_bytes_per_s: initial,
min_pacing_bytes_per_s: RQ_MIN_PACING_BPS,
max_pacing_bytes_per_s: RQ_MAX_PACING_BPS,
initial_cwnd_bytes: symbol_bytes
.saturating_mul(u64::try_from(RQ_ADAPTIVE_BURST_SYMBOLS).unwrap_or(u64::MAX)),
min_cwnd_bytes: symbol_bytes.max(1),
max_cwnd_bytes: RQ_RECEIVER_FLOW_CONTROL_WINDOW_MAX_BYTES,
pacing_gain: 1.0,
cwnd_gain: 2.0,
loss_backoff_threshold: aimd_loss_decrease_threshold(config),
loss_backoff_factor: RQ_AIMD_MULTIPLICATIVE_DECREASE,
loss_delivery_headroom: RQ_LOSS_TARGET_DELIVERY_BACKOFF_HEADROOM,
receiver_window_gain: 1.0,
min_receiver_window_bytes: symbol_bytes.max(1),
max_receiver_window_bytes: RQ_RECEIVER_FLOW_CONTROL_WINDOW_MAX_BYTES,
min_rtt_window_micros: 10_000_000,
}
}
impl RqDeliverySampleKind {
fn feeds_pacing_estimator(self) -> bool {
matches!(self, Self::InitialOrRepair)
}
}
fn delivery_sample_kind_for_need_more_response(
requested_sources: usize,
fec_fallback_active: bool,
) -> RqDeliverySampleKind {
if requested_sources == 0 || fec_fallback_active {
RqDeliverySampleKind::InitialOrRepair
} else {
RqDeliverySampleKind::SourceRetransmit
}
}
#[allow(clippy::cast_precision_loss, clippy::cast_possible_truncation)]
impl RqAdaptiveSendState {
fn new(seed: u64, config: &RqConfig, fanout: usize) -> Self {
let fixed_k = fixed_block_k(config);
let cores = std::thread::available_parallelism().map_or(4.0, |n| {
f64::from(u32::try_from(n.get()).unwrap_or(u32::MAX))
});
let policy = AdaptivePolicy {
cores,
min_samples_to_activate: RQ_ADAPTIVE_MIN_SAMPLES,
arm_grid_k: vec![fixed_k],
arm_grid_fanout: vec![fanout.max(1)],
..AdaptivePolicy::default()
};
let est = PathEstimate {
coding_ref_k: fixed_k,
dec_symbols_per_s: RQ_ASSUMED_DECODE_SYMBOLS_PER_S,
enc_symbols_per_s: RQ_ASSUMED_DECODE_SYMBOLS_PER_S,
coding_gamma: RQ_CODING_GAMMA,
..PathEstimate::unknown()
};
let mut controller = AdaptiveController::new(policy, seed);
controller.update_estimate(est);
Self {
controller,
shared_rate: DatagramRateController::new(rq_shared_rate_config(config)),
shared_rate_decision: None,
shared_rate_clock_micros: 0,
loss_detector: AtpLossDetector::new(),
beacons: BeaconScheduler::new(seed, Instant::now()),
est,
symbol_size: config.symbol_size,
aimd_rate_bps: round0_bad_link_pacing_bps(config).unwrap_or(RQ_COLD_START_PACING_BPS),
aimd_feedback_seen: false,
last_round_loss_fraction: 0.0,
loss_ema: 0.0,
pacing_loss_ema: 0.0,
pacing_loss_bar: 0.0,
loss_bar: 0.0,
bw_ema_bps: 0.0,
bw_trough_bps: 0.0,
loss_pacing_cap_bps: None,
loss_fec_floor: 0.0,
regime_shift: false,
last_pending_rank: None,
}
}
fn record_beacon_exchange(&mut self, control_wait: Duration) {
let now = Instant::now();
let measurement = BeaconMeasurement::with_rtt(duration_micros_u32(control_wait), 0);
let _action = self.beacons.next_action(now, measurement);
self.beacons.observe_probe_result(now, control_wait);
}
fn mark_control_peer_activity(&mut self) {
self.beacons.mark_peer_activity(Instant::now());
}
fn next_control_keepalive_due(&mut self) -> bool {
let measurement = self
.beacons
.latest_rtt()
.map_or_else(BeaconMeasurement::empty, |rtt| {
BeaconMeasurement::with_rtt(duration_micros_u32(rtt), 0)
});
self.beacons
.next_action(Instant::now(), measurement)
.is_some()
}
fn control_liveness_expired(&self) -> bool {
self.beacons.peer_liveness_expired()
}
fn missed_control_probes(&self) -> u8 {
self.beacons.missed_probes()
}
fn round_tuning(&mut self, config: &RqConfig) -> RqRoundTuning {
let fixed = RqRoundTuning {
repair_overhead: config.repair_overhead.max(1.0),
pacing: RqSprayPacing::cold_start(config.symbol_size),
};
let Some(mut plan) = self.controller.next_block_plan(self.symbol_size) else {
return fixed;
};
// Bound the wire-loss-driven overhead before it feeds either the repair budget or the
// pacing rate, so a round-0 over-pace artifact can't blow it up to ~10× (MATRIX-12).
plan.overhead = plan.overhead.min(RQ_MAX_ROUND_REPAIR_OVERHEAD);
let mut repair_overhead = config
.repair_overhead
.max(1.0 + plan.overhead)
.max(1.0 + self.loss_fec_floor);
let model_rate = self.pacing_rate_for(plan, config);
let mut rate = if self.aimd_feedback_seen {
self.aimd_rate_bps
} else {
model_rate.min(self.aimd_rate_bps)
};
if let Some(cap) = self.loss_pacing_cap_bps {
rate = rate.min(self.loss_pacing_cap_for_current_regime(cap, config));
}
if let Some(decision) = self.shared_rate_decision {
rate = rate.min(
decision
.pacing_bytes_per_s
.max(aimd_decrease_floor_bps(config)),
);
}
if self.regime_shift || self.pacing_loss_bar >= RQ_REGIME_SHIFT_LOSS_DELTA {
repair_overhead = repair_overhead.max(1.03);
if !self.aimd_feedback_seen {
rate = rate.min(RQ_COLD_START_PACING_BPS / 2);
}
}
RqRoundTuning {
repair_overhead,
pacing: RqSprayPacing::from_rate(
rate,
config.symbol_size,
RQ_ADAPTIVE_BURST_SYMBOLS,
Some(duration_from_secs(self.est.rtt_s)),
self.pacing_loss_ema > 0.0,
),
}
}
fn shared_rate_decision(&self) -> Option<DatagramRateDecision> {
self.shared_rate_decision
}
fn round0_tuning(&mut self, config: &RqConfig) -> RqRoundTuning {
let mut tuning = self.round_tuning(config);
let round0_repair_overhead = round0_loss_target_repair_overhead(config);
if let Some(rate) = round0_bad_link_pacing_bps(config) {
tuning
.pacing
.set_rate_bytes_per_sec(rate, RQ_MAX_PACING_BPS);
}
if round0_loss_target_repair_enabled(config) {
rqtrace!(
"sender: round0_loss_target_tuning loss_target={:.4} loss_bar={:.4} source_k={} repair_overhead={:.4} extra_fraction={:.4} pacing_rate_Bps={} path_rate_bps={} max_block_size={} symbol_size={}",
config.round0_loss_target,
round0_loss_target_loss_bar(config),
fixed_block_k(config),
round0_repair_overhead,
(round0_repair_overhead - 1.0).max(0.0),
tuning.pacing.rate_bytes_per_sec(),
tuning.pacing.path_rate_bps,
config.max_block_size,
config.symbol_size,
);
}
tuning.repair_overhead = tuning.repair_overhead.max(round0_repair_overhead);
tuning
}
fn source_fec_fallback_tuning(&mut self, config: &RqConfig) -> RqRoundTuning {
let mut tuning = self.round_tuning(config);
let k = fixed_block_k(config);
let loss_bar = self.source_fec_fallback_loss_bar(config);
let overhead = adaptive::overhead_for_target(
k,
loss_bar,
RQ_SOURCE_FEC_FALLBACK_ALPHA,
RQ_SOURCE_FEC_FALLBACK_MAX_OVERHEAD,
);
let measured_loss_overhead =
measured_feedback_repair_overhead(self.last_round_loss_fraction);
tuning.repair_overhead = tuning
.repair_overhead
.max(1.0 + overhead)
.max(1.0 + measured_loss_overhead)
.max(1.0 + RQ_SOURCE_FEC_FALLBACK_MIN_OVERHEAD)
// Bound the FEC-fallback budget the same as round_tuning: when the wire-loss
// estimate is inflated by a round-0 over-pace artifact (loss_bar≈0.9),
// overhead_for_target returns ~9.7 ⇒ ~10.7× and a single round sprays ~10× the
// object (~518MB for 50M → 1.7GB recv RSS). Cap at ≤2× total so repair stays
// bounded; convergence still completes in ~2 rounds for realistic loss (MATRIX-13).
.min(1.0 + RQ_MAX_ROUND_REPAIR_OVERHEAD);
tuning
}
fn source_fec_fallback_loss_bar(&self, config: &RqConfig) -> f64 {
let configured_loss_bar = if round0_loss_target_repair_enabled(config) {
round0_loss_target_loss_bar(config)
} else {
0.0
};
let measured_loss = self.pacing_loss_ema.max(self.est.loss_p_hat);
self.loss_bar
.max(self.loss_ema)
.max(measured_loss)
.max(configured_loss_bar)
.max(RQ_SOURCE_FEC_FALLBACK_MIN_LOSS_BAR)
}
#[cfg(test)]
fn observe_need_more(
&mut self,
config: &RqConfig,
digests: &[EntryDigest],
pending: &BTreeSet<u32>,
sent_this_round: u64,
received_this_round: u64,
round_loss_fraction: Option<f64>,
delivery_sample_kind: RqDeliverySampleKind,
send_wall: Duration,
control_wait: Duration,
total_bytes: u64,
) {
let pending_bytes = pending_bytes(digests, pending);
self.observe_need_more_with_progress(
config,
digests,
pending,
pending_bytes,
RqNeedMoreProgress::default(),
sent_this_round,
received_this_round,
round_loss_fraction,
delivery_sample_kind,
send_wall,
control_wait,
total_bytes,
);
}
fn observe_need_more_with_progress(
&mut self,
config: &RqConfig,
digests: &[EntryDigest],
pending: &BTreeSet<u32>,
prior_pending_bytes: u64,
progress: RqNeedMoreProgress,
sent_this_round: u64,
received_this_round: u64,
round_loss_fraction: Option<f64>,
delivery_sample_kind: RqDeliverySampleKind,
send_wall: Duration,
control_wait: Duration,
total_bytes: u64,
) {
self.record_beacon_exchange(control_wait);
let send_wall_s = finite_duration_s(send_wall);
let rtt_s = finite_duration_s(control_wait);
let pending_bytes = pending_bytes(digests, pending);
let sent_symbols = sent_this_round.max(1);
let pending_units = u64::try_from(pending.len()).unwrap_or(u64::MAX).max(1);
let received_symbols = received_this_round.min(sent_symbols);
let decode_pending_loss = (pending_units as f64 / sent_symbols as f64).clamp(0.0, 0.90);
let derived_wire_loss = if sent_this_round == 0 {
0.0
} else {
(1.0 - received_symbols as f64 / sent_symbols as f64).clamp(0.0, 0.90)
};
let wire_loss_hat = round_loss_fraction
.filter(|loss| loss.is_finite())
.map_or(derived_wire_loss, |loss| loss.clamp(0.0, 0.90));
let feeds_pacing_estimator = delivery_sample_kind.feeds_pacing_estimator();
let estimator_wire_loss = if feeds_pacing_estimator {
wire_loss_hat
} else {
0.0
};
let symbol_payload_bytes = u64::from(config.symbol_size.max(1));
let sent_payload_bytes = sent_symbols.saturating_mul(symbol_payload_bytes);
let useful_bytes = received_symbols.saturating_mul(symbol_payload_bytes);
let receiver_delivery_bps = (useful_bytes as f64 / send_wall_s).max(1.0);
let offered_bps = (sent_payload_bytes as f64 / send_wall_s).max(1.0);
let progress_delivery_bps = self.progress_delivery_bps(
config,
prior_pending_bytes,
pending_bytes,
progress,
symbol_payload_bytes,
send_wall_s,
);
let progress_delivery_bytes = progress_delivery_bps.map(|delivery_bps| {
(delivery_bps * send_wall_s)
.ceil()
.clamp(0.0, sent_payload_bytes as f64) as u64
});
// Rank-stall reads as congestion ONLY when arrivals corroborate it.
// The stall-ratio congestion proxy exists for kernel-overflow drops,
// but overflow also depresses the ARRIVAL count — whereas a
// decode-side stall (e.g. one rank-deficient block) leaves arrivals
// healthy. Without this gate a single stalled block inflated the
// pacing loss to 0.90 and halved the path rate every round while
// 90%+ of symbols were arriving (MATRIX-8 re-entry via the
// progress-stall term; MATRIX-207). Decode pressure still feeds
// repair/FEC sizing below, never the pacer.
let arrival_ratio = received_symbols as f64 / sent_symbols.max(1) as f64;
let arrivals_corroborate_congestion =
arrival_ratio < 1.0 - aimd_loss_decrease_threshold(config);
let progress_congestion_loss = if arrivals_corroborate_congestion {
progress_delivery_bps
.and_then(|delivery_bps| {
loss_target_progress_congestion_loss(config, delivery_bps, offered_bps)
})
.unwrap_or(0.0)
} else {
0.0
};
let pacing_loss_sample = estimator_wire_loss.max(progress_congestion_loss);
// Same evidence rule for the delivery estimate the AIMD backs off
// toward: with healthy arrivals, the arrival-clocked receiver rate is
// the path's true delivery; the rank-progress rate only substitutes
// when arrivals themselves say the wire is failing.
let observed_delivery_bps = if arrivals_corroborate_congestion {
progress_delivery_bps.unwrap_or(receiver_delivery_bps)
} else {
receiver_delivery_bps
}
.max(1.0);
let delivered_payload_bytes = progress_delivery_bytes
.unwrap_or(useful_bytes)
.min(sent_payload_bytes);
self.observe_shared_rate(
config,
sent_payload_bytes,
delivered_payload_bytes,
pending_bytes,
send_wall,
control_wait,
);
if sent_this_round != 0 && feeds_pacing_estimator {
self.apply_aimd_feedback(config, pacing_loss_sample, Some(observed_delivery_bps));
}
let byte_pressure = if total_bytes == 0 {
0.0
} else {
(pending_bytes as f64 / total_bytes as f64).clamp(0.0, 1.0)
};
let pressure_loss = byte_pressure * RQ_PENDING_PRESSURE_LOSS_FLOOR;
let repair_loss_hat = pacing_loss_sample
.max(decode_pending_loss)
.max(pressure_loss)
.clamp(0.0, 0.90);
if feeds_pacing_estimator {
self.regime_shift = self.pacing_loss_ema > 0.0
&& pacing_loss_sample > (self.pacing_loss_ema * 3.0 + RQ_REGIME_SHIFT_LOSS_DELTA);
}
self.loss_ema = ema(self.loss_ema, repair_loss_hat, RQ_LOSS_EMA_ALPHA);
if feeds_pacing_estimator {
self.pacing_loss_ema = ema(self.pacing_loss_ema, pacing_loss_sample, RQ_LOSS_EMA_ALPHA);
}
let raw_loss_bar = repair_loss_hat.max(self.loss_ema) * RQ_LOSS_BAR_MULTIPLIER;
self.loss_bar = if self.loss_bar <= 0.0 {
raw_loss_bar
} else {
ema(self.loss_bar, raw_loss_bar, RQ_LOSS_EMA_ALPHA).max(repair_loss_hat)
}
.clamp(0.0, 0.90);
if feeds_pacing_estimator {
let raw_pacing_loss_bar =
pacing_loss_sample.max(self.pacing_loss_ema) * RQ_LOSS_BAR_MULTIPLIER;
self.pacing_loss_bar = if self.pacing_loss_bar <= 0.0 {
raw_pacing_loss_bar
} else {
ema(self.pacing_loss_bar, raw_pacing_loss_bar, RQ_LOSS_EMA_ALPHA)
.max(pacing_loss_sample)
}
.clamp(0.0, 0.90);
}
let useful_factor = (1.0 - pacing_loss_sample * 0.5).clamp(0.25, 1.0);
let bw_sample = offered_bps * useful_factor;
let sent_payload_fraction = if pending_bytes == 0 {
1.0
} else {
(sent_payload_bytes as f64 / pending_bytes as f64).clamp(0.0, 1.0)
};
let stalled_repair_sample = byte_pressure >= RQ_STALLED_REPAIR_PRESSURE_MIN
&& sent_payload_fraction < RQ_STALLED_REPAIR_PAYLOAD_FRACTION_MAX
&& pacing_loss_sample <= RQ_MILD_LOSS_PACING_MAX_LOSS;
if feeds_pacing_estimator && (self.bw_ema_bps <= 0.0 || !stalled_repair_sample) {
self.bw_ema_bps = if self.bw_ema_bps <= 0.0 {
bw_sample
} else {
ema(self.bw_ema_bps, bw_sample, RQ_BW_EMA_ALPHA)
};
self.update_bw_trough(bw_sample);
}
self.est = PathEstimate {
rtt_s,
loss_p_hat: self.pacing_loss_ema,
loss_p_bar: self.loss_bar,
bw_median_bps: self.bw_ema_bps,
bw_trough_bps: self.bw_trough_bps.max(self.bw_ema_bps * 0.5),
enc_symbols_per_s: RQ_ASSUMED_DECODE_SYMBOLS_PER_S,
dec_symbols_per_s: RQ_ASSUMED_DECODE_SYMBOLS_PER_S,
coding_ref_k: fixed_block_k(config),
coding_gamma: RQ_CODING_GAMMA,
samples: self.est.samples.saturating_add(1),
};
self.controller.update_estimate(self.est);
let cwnd_bytes = (self.bw_ema_bps * rtt_s)
.max(f64::from(config.symbol_size.max(1)))
.ceil() as u64;
self.loss_pacing_cap_bps = None;
self.loss_fec_floor = 0.0;
let lost_symbols = ((sent_symbols as f64) * pacing_loss_sample)
.ceil()
.clamp(0.0, sent_symbols as f64) as u64;
if feeds_pacing_estimator {
let loss_result = self.loss_detector.observe_datagram_loss_sample(
sent_symbols,
lost_symbols,
Some(control_wait),
sent_payload_bytes,
cwnd_bytes,
);
// "Mild" is relative to the configured regime: on a loss-target
// link the expected erasure rate (plus the AIMD margin) is the
// ambient condition, not congestion (MATRIX-207).
let mild_loss_ceiling =
aimd_loss_decrease_threshold(config).max(RQ_MILD_LOSS_PACING_MAX_LOSS);
let mild_wire_loss = pacing_loss_sample <= mild_loss_ceiling
&& self.pacing_loss_ema <= mild_loss_ceiling;
self.apply_loss_recommendations(&loss_result.recommendations, mild_wire_loss);
self.controller.observe_path_signals(
sent_symbols,
received_symbols,
send_wall_s,
useful_bytes,
config.symbol_size,
PathSignalSample {
smoothed_rtt_s: rtt_s,
congestion_window_bytes: cwnd_bytes.max(u64::from(config.symbol_size.max(1))),
loss_rate: pacing_loss_sample,
},
);
}
}
fn observe_shared_rate(
&mut self,
config: &RqConfig,
sent_payload_bytes: u64,
delivered_payload_bytes: u64,
pending_bytes: u64,
send_wall: Duration,
control_wait: Duration,
) {
if sent_payload_bytes == 0 {
return;
}
let rtt_micros = u64::try_from(control_wait.as_micros())
.unwrap_or(u64::MAX)
.max(1);
if self.shared_rate_clock_micros == 0 {
self.shared_rate_clock_micros = 1;
let _ = self.shared_rate.observe(DatagramRateSample {
now_micros: self.shared_rate_clock_micros,
sent_bytes: 1,
acked_bytes: 1,
lost_bytes: 0,
bytes_in_flight: 0,
rtt_micros: Some(rtt_micros),
receiver_credit_bytes: None,
receiver_window_bytes: None,
});
}
let elapsed_micros = u64::try_from(send_wall.as_micros())
.unwrap_or(u64::MAX)
.max(1);
self.shared_rate_clock_micros =
self.shared_rate_clock_micros.saturating_add(elapsed_micros);
let receiver_credit = rq_receiver_flow_credit_bytes(config, pending_bytes);
let sample_sent_bytes = sent_payload_bytes.max(1);
let sample_delivered_bytes = delivered_payload_bytes.min(sample_sent_bytes);
let bytes_in_flight = sample_sent_bytes.saturating_sub(sample_delivered_bytes);
self.shared_rate_decision = Some(self.shared_rate.observe(DatagramRateSample {
now_micros: self.shared_rate_clock_micros,
sent_bytes: sample_sent_bytes,
acked_bytes: sample_delivered_bytes,
lost_bytes: bytes_in_flight,
bytes_in_flight,
rtt_micros: Some(rtt_micros),
receiver_credit_bytes: Some(receiver_credit),
receiver_window_bytes: Some(receiver_credit.saturating_add(bytes_in_flight)),
}));
}
fn progress_delivery_bps(
&mut self,
config: &RqConfig,
prior_pending_bytes: u64,
pending_bytes: u64,
progress: RqNeedMoreProgress,
symbol_payload_bytes: u64,
send_wall_s: f64,
) -> Option<f64> {
if !round0_loss_target_repair_enabled(config) {
self.last_pending_rank = progress.pending_rank;
return None;
}
let completed_entry_bytes = prior_pending_bytes.saturating_sub(pending_bytes);
let progress_accounted = progress.pending_rank.is_some()
|| progress.pending_rank_columns.is_some()
|| progress.pending_rank_deficit.is_some()
|| progress.pending_decode_jobs.is_some();
let rank_delta_bytes = progress.pending_rank.map(|rank| {
let delta = self
.last_pending_rank
.map_or(rank, |previous| rank.saturating_sub(previous));
self.last_pending_rank = Some(rank);
delta.saturating_mul(symbol_payload_bytes)
});
let progress_bytes = completed_entry_bytes.max(rank_delta_bytes.unwrap_or(0));
if progress_bytes == 0 {
progress_accounted.then_some(0.0)
} else {
Some(progress_bytes as f64 / send_wall_s)
}
}
fn apply_aimd_feedback(
&mut self,
config: &RqConfig,
loss_fraction: f64,
observed_delivery_bps: Option<f64>,
) {
let loss = loss_fraction.clamp(0.0, 0.90);
let decrease_threshold = aimd_loss_decrease_threshold(config);
self.aimd_feedback_seen = true;
self.last_round_loss_fraction = loss;
if loss > decrease_threshold {
let multiplicative =
(self.aimd_rate_bps as f64 * RQ_AIMD_MULTIPLICATIVE_DECREASE).ceil() as u64;
let reduced = loss_target_delivery_backoff_bps(config, observed_delivery_bps)
.map_or(multiplicative, |delivery_backoff| {
delivery_backoff.min(multiplicative)
});
self.aimd_rate_bps = reduced.clamp(aimd_decrease_floor_bps(config), RQ_MAX_PACING_BPS);
} else if loss <= RQ_AIMD_CLEAN_INCREASE_THRESHOLD
&& let Some(ceiling) = aimd_clean_increase_ceiling_bps(config)
&& self.aimd_rate_bps < ceiling
{
self.aimd_rate_bps = self
.aimd_rate_bps
.saturating_add(RQ_AIMD_ADDITIVE_INCREASE_BYTES_PER_S)
.clamp(RQ_MIN_PACING_BPS, ceiling);
}
}
fn apply_loss_recommendations(
&mut self,
recommendations: &[LossRecommendation],
mild_wire_loss: bool,
) {
for recommendation in recommendations {
match recommendation {
// Wire-slowing recommendations only apply when loss EXCEEDS the
// regime's expectation (`mild_wire_loss` is threshold-aware):
// on a configured-lossy link the erasure loss is the ambient
// condition FEC already pays for, not congestion evidence.
// Un-gated, sustained 9% link loss halved the pacing cap to
// the floor every round and turned a 66s repair round into
// 209s (500M/broken, MATRIX-207).
LossRecommendation::ReduceCongestionWindow { factor } if !mild_wire_loss => {
let cap = (self.bw_ema_bps * (*factor).clamp(0.1, 1.0)).ceil() as u64;
self.lower_pacing_cap(cap);
}
LossRecommendation::EnablePacing { rate } if !mild_wire_loss => {
let ema_cap = if self.bw_ema_bps > 0.0 {
(self.bw_ema_bps * 0.75).ceil() as u64
} else {
RQ_COLD_START_PACING_BPS / 2
};
self.lower_pacing_cap((*rate).max(ema_cap));
}
LossRecommendation::ReduceCongestionWindow { .. }
| LossRecommendation::EnablePacing { .. } => {}
LossRecommendation::EnableFec { rate } => {
self.loss_fec_floor = self.loss_fec_floor.max((*rate).clamp(0.0, 0.50));
}
LossRecommendation::SwitchCongestionControl { .. } if !mild_wire_loss => {
self.regime_shift = true;
let cap = if self.bw_ema_bps > 0.0 {
(self.bw_ema_bps * 0.5).ceil() as u64
} else {
RQ_COLD_START_PACING_BPS / 2
};
self.lower_pacing_cap(cap);
self.loss_fec_floor = self.loss_fec_floor.max(0.03);
}
LossRecommendation::SwitchCongestionControl { .. } => {}
LossRecommendation::IncreaseReorderingThreshold { .. } => {}
}
}
}
fn lower_pacing_cap(&mut self, cap_bps: u64) {
let cap = cap_bps.clamp(RQ_MIN_PACING_BPS, RQ_MAX_PACING_BPS);
self.loss_pacing_cap_bps = Some(
self.loss_pacing_cap_bps
.map_or(cap, |previous| previous.min(cap)),
);
}
fn observe_probe_success(
&mut self,
config: &RqConfig,
sent_this_round: u64,
send_wall: Duration,
control_wait: Duration,
) {
self.record_beacon_exchange(control_wait);
if sent_this_round == 0 {
self.est = PathEstimate {
rtt_s: finite_duration_s(control_wait),
samples: self.est.samples.saturating_add(1),
..self.est
};
self.controller.update_estimate(self.est);
return;
}
let send_wall_s = finite_duration_s(send_wall);
let rtt_s = finite_duration_s(control_wait);
let sent_payload_bytes =
sent_this_round.saturating_mul(u64::from(config.symbol_size.max(1)));
let bw_sample = (sent_payload_bytes as f64 / send_wall_s).max(1.0);
self.observe_shared_rate(
config,
sent_payload_bytes,
sent_payload_bytes,
sent_payload_bytes,
send_wall,
control_wait,
);
self.apply_aimd_feedback(config, 0.0, Some(bw_sample));
self.bw_ema_bps = if self.bw_ema_bps <= 0.0 {
bw_sample
} else {
ema(self.bw_ema_bps, bw_sample, RQ_BW_EMA_ALPHA)
};
self.update_bw_trough(bw_sample);
self.loss_ema = ema(self.loss_ema, 0.0, RQ_LOSS_EMA_ALPHA);
self.pacing_loss_ema = ema(self.pacing_loss_ema, 0.0, RQ_LOSS_EMA_ALPHA);
self.loss_bar = ema(self.loss_bar, 0.0, RQ_LOSS_EMA_ALPHA);
self.pacing_loss_bar = ema(self.pacing_loss_bar, 0.0, RQ_LOSS_EMA_ALPHA);
self.loss_pacing_cap_bps = None;
self.loss_fec_floor = 0.0;
self.est = PathEstimate {
rtt_s,
loss_p_hat: self.pacing_loss_ema,
loss_p_bar: self.loss_bar,
bw_median_bps: self.bw_ema_bps,
bw_trough_bps: self.bw_trough_bps.max(self.bw_ema_bps * 0.5),
enc_symbols_per_s: RQ_ASSUMED_DECODE_SYMBOLS_PER_S,
dec_symbols_per_s: RQ_ASSUMED_DECODE_SYMBOLS_PER_S,
coding_ref_k: fixed_block_k(config),
coding_gamma: RQ_CODING_GAMMA,
samples: self.est.samples.saturating_add(1),
};
self.controller.update_estimate(self.est);
let cwnd_bytes = (self.bw_ema_bps * rtt_s)
.max(f64::from(config.symbol_size.max(1)))
.ceil() as u64;
self.controller.observe_path_signals(
sent_this_round,
sent_this_round,
send_wall_s,
sent_payload_bytes,
config.symbol_size,
PathSignalSample {
smoothed_rtt_s: rtt_s,
congestion_window_bytes: cwnd_bytes.max(u64::from(config.symbol_size.max(1))),
loss_rate: 0.0,
},
);
}
fn pacing_rate_for(&self, plan: BlockPlan, config: &RqConfig) -> u64 {
let mut network_bps = if self.est.bw_median_bps > 0.0 {
self.est.bw_median_bps.min(self.est.bw_trough_bps.max(1.0))
} else {
RQ_COLD_START_PACING_BPS as f64
};
if self.mild_loss_pacing_floor_applies() {
network_bps = network_bps.max(self.mild_loss_pacing_floor_bps(config) as f64);
}
let decode_bps =
self.est.decode_symbols_per_s_at(plan.k) * f64::from(self.symbol_size.max(1));
let base = network_bps.min(decode_bps.max(1.0));
let rate = base / (1.0 + plan.overhead.max(0.0));
rqtrace!(
"pacing_rate_for: network_bps={:.0} decode_bps={:.0} base={:.0} overhead={:.4} rate={:.0} bw_median={:.0} bw_trough={:.0} mild_floor={}",
network_bps,
decode_bps,
base,
plan.overhead.max(0.0),
rate,
self.est.bw_median_bps,
self.est.bw_trough_bps,
self.mild_loss_pacing_floor_applies()
);
rate.ceil()
.clamp(RQ_MIN_PACING_BPS as f64, RQ_MAX_PACING_BPS as f64) as u64
}
fn update_bw_trough(&mut self, bw_sample: f64) {
if self.bw_trough_bps <= 0.0 || bw_sample < self.bw_trough_bps {
self.bw_trough_bps = bw_sample;
} else {
self.bw_trough_bps = ema(self.bw_trough_bps, bw_sample, RQ_BW_TROUGH_RECOVERY_ALPHA)
.min(self.bw_ema_bps.max(bw_sample));
}
}
fn mild_loss_pacing_floor_applies(&self) -> bool {
let pacing_loss = self.pacing_loss_ema;
let has_repair_pressure = self.loss_bar > 0.0 || self.pacing_loss_bar > 0.0;
!self.regime_shift
&& has_repair_pressure
&& pacing_loss <= RQ_MILD_LOSS_PACING_MAX_LOSS
&& self.est.bw_median_bps > 0.0
}
fn mild_loss_pacing_floor_bps(&self, config: &RqConfig) -> u64 {
if let Some(rate) = round0_bad_link_pacing_bps(config) {
return rate;
}
let floor_fraction = if round0_source_first_loss_target(config)
&& config.round0_loss_target > f64::EPSILON
{
RQ_SOURCE_FIRST_MILD_LOSS_PACING_FLOOR_FRACTION
} else {
RQ_MILD_LOSS_PACING_FLOOR_FRACTION
};
(RQ_COLD_START_PACING_BPS as f64 * floor_fraction).ceil() as u64
}
fn loss_pacing_cap_for_current_regime(&self, cap: u64, config: &RqConfig) -> u64 {
if self.mild_loss_pacing_floor_applies() {
cap.max(self.mild_loss_pacing_floor_bps(config))
} else {
cap
}
}
}
fn fixed_block_k(config: &RqConfig) -> u32 {
let symbol_size = usize::from(config.symbol_size.max(1));
let k = config.max_block_size.div_ceil(symbol_size).max(1);
u32::try_from(k).unwrap_or(u32::MAX)
}
/// Per-block decode-failure target for the round-0 first flight.
///
/// Deliberately far looser than `RQ_SOURCE_FEC_FALLBACK_ALPHA` (1e-6): a
/// round-0 block that misses decode is simply repaired by the feedback round
/// at the (now sustained) path rate, so paying a ~4.75σ concentration margin
/// up front is a pure byte tax. On the 500M/broken cell the 1e-6 target
/// inflated round-0 to +25.3% repair overhead — 657 MB on a 10 mbit link is
/// ≥526 s of wire time before any repair, which alone loses to rsync. At
/// α=0.02, K=437 the margin drops to ~+16-18% and the expected ~2% of blocks
/// that miss cost one cheap repair round (MATRIX-207).
const RQ_ROUND0_TARGET_ALPHA: f64 = 0.02;
fn round0_loss_target_repair_overhead(config: &RqConfig) -> f64 {
if !round0_loss_target_repair_enabled(config) {
return config.repair_overhead.max(1.0);
}
let loss_bar = round0_loss_target_loss_bar(config);
let overhead = adaptive::decode_repair_overhead_for_target(
fixed_block_k(config),
loss_bar,
RQ_ROUND0_TARGET_ALPHA,
RQ_SOURCE_FEC_FALLBACK_MAX_OVERHEAD,
)
.min(RQ_MAX_ROUND_REPAIR_OVERHEAD);
config.repair_overhead.max(1.0 + overhead)
}
fn round0_loss_target_loss_bar(config: &RqConfig) -> f64 {
(config.round0_loss_target * (1.0 + RQ_ROUND0_TARGET_LOSS_MARGIN_FRACTION)
+ RQ_ROUND0_TARGET_LOSS_MARGIN_MIN)
.clamp(0.0, RQ_SOURCE_FEC_FALLBACK_MAX_OVERHEAD)
}
fn round0_loss_target_repair_enabled(config: &RqConfig) -> bool {
let loss = config.round0_loss_target;
loss.is_finite() && loss >= RQ_ROUND0_TARGET_LOSS_ENABLE_MIN
}
fn round0_bad_link_pacing_bps(config: &RqConfig) -> Option<u64> {
let loss = config.round0_loss_target;
if loss.is_finite()
&& (RQ_BAD_LINK_ROUND0_LOSS_MIN..=RQ_BAD_LINK_ROUND0_LOSS_MAX).contains(&loss)
{
Some(RQ_BAD_LINK_ROUND0_PACING_BPS)
} else if loss.is_finite()
&& loss > RQ_BROKEN_LINK_ROUND0_LOSS_MIN
&& loss <= RQ_BROKEN_LINK_ROUND0_LOSS_MAX
{
Some(RQ_BROKEN_LINK_ROUND0_PACING_BPS)
} else {
None
}
}
fn round0_source_first_loss_target(config: &RqConfig) -> bool {
let loss = config.round0_loss_target;
loss.is_finite() && loss >= 0.0 && !round0_loss_target_repair_enabled(config)
}
fn small_clean_source_only_round0(total_bytes: u64, config: &RqConfig) -> bool {
clean_control_source_stream_round0(config)
&& total_bytes <= RQ_SMALL_CLEAN_SOURCE_ONLY_MAX_BYTES
}
fn clean_control_source_stream_round0(config: &RqConfig) -> bool {
let loss_free_target = round0_source_first_loss_target(config)
&& (0.0..=f64::EPSILON).contains(&config.round0_loss_target);
control_source_stream_base_round0(config) && loss_free_target
}
fn near_clean_control_source_stream_round0(config: &RqConfig) -> bool {
let source_first_target = round0_source_first_loss_target(config)
&& config.round0_loss_target <= RQ_CONTROL_SOURCE_STREAM_MAX_LOSS_TARGET + f64::EPSILON;
control_source_stream_base_round0(config) && source_first_target
}
fn control_source_stream_base_round0(config: &RqConfig) -> bool {
config.debug_drop_one_in == 0
&& config.repair_overhead.is_finite()
&& config.repair_overhead <= RQ_SMALL_CLEAN_SOURCE_ONLY_MAX_REPAIR_OVERHEAD
}
fn control_source_stream_eligible(total_bytes: u64, config: &RqConfig) -> bool {
if total_bytes == 0 || total_bytes > config.max_transfer_bytes {
return false;
}
if near_clean_control_source_stream_round0(config) {
return true;
}
// ADDITIVE composing path selection (MATRIX-199 / 317hxr.2.5): a LARGE object over a
// MODERATELY-lossy link takes the reliable control-source stream instead of the FEC datagram
// spray, because the spray rate-collapses on large lossy objects (a non-converging block
// inflates the loss estimate → the pacing rate halves each round → timeout) while reliable TCP
// retransmit completes and beats rsync (500M/bad@2%: reliable-stream 95.3s vs rsync 97.9s;
// spray times out). Small objects and high-loss links still take the spray, where forward-repair
// wins (e.g. 50M/broken). This does NOT touch the FEC pacing path; once the spray collapse is
// fixed (317hxr.2.5) this fallback can be narrowed.
total_bytes >= RQ_LARGE_LOSSY_SOURCE_STREAM_MIN_BYTES
&& control_source_stream_base_round0(config)
&& config.round0_loss_target.is_finite()
&& config.round0_loss_target >= 0.0
&& config.round0_loss_target <= RQ_LARGE_LOSSY_SOURCE_STREAM_MAX_LOSS_TARGET
}
fn apply_small_clean_round0_source_only(
total_bytes: u64,
config: &RqConfig,
mut tuning: RqRoundTuning,
) -> RqRoundTuning {
if small_clean_source_only_round0(total_bytes, config) {
tuning.repair_overhead = 1.0;
tuning.pacing = RqSprayPacing::cold_start(config.symbol_size);
}
tuning
}
fn measured_feedback_repair_overhead(loss_fraction: f64) -> f64 {
if !loss_fraction.is_finite() {
return 0.0;
}
let loss = loss_fraction.clamp(0.0, RQ_SOURCE_FEC_FALLBACK_MAX_OVERHEAD);
if loss < RQ_FEEDBACK_REPAIR_LOSS_ENABLE_MIN {
return 0.0;
}
(loss * (1.0 + RQ_FEEDBACK_REPAIR_LOSS_MARGIN_FRACTION) + RQ_FEEDBACK_REPAIR_LOSS_MARGIN_MIN)
.clamp(0.0, RQ_SOURCE_FEC_FALLBACK_MAX_OVERHEAD)
}
fn aimd_loss_decrease_threshold(config: &RqConfig) -> f64 {
let expected_loss = if round0_loss_target_repair_enabled(config) {
config.round0_loss_target
} else {
0.0
};
(expected_loss + RQ_AIMD_LOSS_DECREASE_THRESHOLD_MARGIN)
.max(RQ_AIMD_LOSS_DECREASE_THRESHOLD_MIN)
.clamp(0.0, 0.90)
}
#[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)]
fn loss_target_delivery_backoff_bps(
config: &RqConfig,
observed_delivery_bps: Option<f64>,
) -> Option<u64> {
if !round0_loss_target_repair_enabled(config) {
return None;
}
let delivery_bps = observed_delivery_bps?.clamp(1.0, RQ_MAX_PACING_BPS as f64);
if !delivery_bps.is_finite() {
return None;
}
Some(
(delivery_bps * RQ_LOSS_TARGET_DELIVERY_BACKOFF_HEADROOM)
.ceil()
.clamp(RQ_MIN_PACING_BPS as f64, RQ_MAX_PACING_BPS as f64) as u64,
)
}
fn aimd_decrease_floor_bps(config: &RqConfig) -> u64 {
if round0_source_first_loss_target(config) && config.round0_loss_target > f64::EPSILON {
// MATRIX-77: the good cell's sparse residual feedback should size
// repair without dragging the next spray below cold-start. Bad/broken
// cells use round-0 repair targets and keep the older AIMD decrease.
RQ_COLD_START_PACING_BPS
} else {
RQ_MIN_PACING_BPS
}
}
fn aimd_clean_increase_ceiling_bps(config: &RqConfig) -> Option<u64> {
if round0_loss_target_repair_enabled(config) {
round0_bad_link_pacing_bps(config)
} else {
Some(RQ_MAX_PACING_BPS)
}
}
fn loss_target_progress_congestion_loss(
config: &RqConfig,
progress_delivery_bps: f64,
offered_bps: f64,
) -> Option<f64> {
if !round0_loss_target_repair_enabled(config) || !offered_bps.is_finite() || offered_bps <= 0.0
{
return None;
}
let delivery_ratio = (progress_delivery_bps / offered_bps).clamp(0.0, 1.0);
if delivery_ratio >= RQ_LOSS_TARGET_PROGRESS_STALL_RATIO {
return None;
}
let decrease_threshold = aimd_loss_decrease_threshold(config);
Some(
(1.0 - delivery_ratio)
.max(decrease_threshold + RQ_LOSS_TARGET_PROGRESS_LOSS_MARGIN)
.clamp(0.0, 0.90),
)
}
fn pending_bytes(digests: &[EntryDigest], pending: &BTreeSet<u32>) -> u64 {
pending.iter().fold(0u64, |acc, index| {
let Some(entry) = usize::try_from(*index)
.ok()
.and_then(|idx| digests.get(idx))
else {
return acc;
};
acc.saturating_add(entry.size)
})
}
fn usize_to_u64(value: usize) -> u64 {
u64::try_from(value).unwrap_or(u64::MAX)
}
fn finite_duration_s(duration: Duration) -> f64 {
duration.as_secs_f64().max(0.000_001)
}
fn duration_from_secs(seconds: f64) -> Duration {
if seconds.is_finite() {
Duration::from_secs_f64(seconds.clamp(0.000_001, 60.0))
} else {
Duration::from_micros(1)
}
}
fn duration_micros_u32(duration: Duration) -> u32 {
u32::try_from(duration.as_micros()).unwrap_or(u32::MAX)
}
fn ema(prev: f64, sample: f64, alpha: f64) -> f64 {
prev.mul_add(1.0 - alpha, sample * alpha)
}
/// Errors from the ATP-over-RaptorQ transport.
#[derive(Debug, thiserror::Error)]
pub enum RqError {
/// Network or local I/O failure.
#[error("io error: {0}")]
Io(#[from] std::io::Error),
/// Frame codec error.
#[error("frame error: {0}")]
Frame(String),
/// JSON (de)serialization error for a control frame.
#[error("control frame decode error: {0}")]
Control(String),
/// The peer rejected the handshake.
#[error("[ASUP-E802] handshake rejected by peer: {0}")]
HandshakeRejected(String),
/// An unexpected frame type arrived for the current protocol state.
#[error("unexpected frame: got {got:?}, expected {expected}")]
Unexpected {
/// The frame type actually received.
got: FrameType,
/// A description of what was expected.
expected: &'static str,
},
/// The transfer exceeded the configured size ceiling.
#[error("transfer exceeds maximum size ({size} > {max} bytes)")]
TooLarge {
/// Declared or observed size.
size: u64,
/// Configured maximum.
max: u64,
},
/// RaptorQ encode/decode error.
#[error("coding error: {0}")]
Coding(String),
/// The fountain feedback loop ran out of rounds with entries still
/// undecoded.
#[error(
"[ASUP-E801] transfer did not converge after {rounds} feedback rounds ({pending} entries still incomplete); if accepted symbols do not advance decode rank, see [ASUP-E805]"
)]
NoConvergence {
/// Feedback rounds attempted.
rounds: u32,
/// Entries still undecoded.
pending: usize,
},
/// Integrity verification failed (SHA-256 or merkle-root mismatch).
#[error("integrity verification failed: {0}")]
Integrity(String),
/// Symbol authentication is missing, mismatched, or invalid.
#[error("symbol authentication failed: {0}")]
Authentication(String),
/// The source path was invalid (missing, unsupported type).
#[error("invalid source path: {0}")]
Source(String),
/// The transfer was cancelled via the capability context.
#[error("transfer cancelled")]
Cancelled,
}
// ─── Wire control payloads (JSON over TCP) ───────────────────────────────────
#[derive(Debug, Clone, Serialize, Deserialize)]
struct Hello {
protocol: u32,
role: String,
peer_id: String,
symbol_size: u16,
max_block_size: u64,
#[serde(default)]
symbol_auth: bool,
/// Total payload bytes of the transfer. The receiver sizes its UDP recv buffer to absorb the
/// sender's (now parallel-encoded) symbol burst so the CPU-bound decode can drain it without
/// kernel drops. `serde(default)` keeps it tolerant of peers that do not send it.
#[serde(default)]
total_bytes: u64,
/// Sender preference for the clean source-only control-stream lane.
///
/// Older receivers ignore this field and omit the matching ack bit, causing
/// the sender to fall back to the UDP/RaptorQ symbol path.
#[serde(default)]
prefer_control_source_stream: bool,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
struct HelloAck {
accepted: bool,
peer_id: String,
/// First UDP port the receiver is listening on for symbol datagrams.
udp_port: u16,
/// Full receiver-side UDP fanout. Empty means legacy single-port `udp_port`.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
udp_ports: Vec<u16>,
/// Receiver accepted the clean source-only control-stream lane.
#[serde(default)]
control_source_stream: bool,
#[serde(skip_serializing_if = "Option::is_none")]
reason: Option<String>,
}
fn hello_ack_udp_ports(ack: &HelloAck) -> SmallVec<[u16; DEFAULT_UDP_FANOUT]> {
let mut ports = SmallVec::<[u16; DEFAULT_UDP_FANOUT]>::new();
if ack.udp_ports.is_empty() {
if ack.udp_port != 0 {
ports.push(ack.udp_port);
}
return ports;
}
if ack.udp_port != 0 {
ports.push(ack.udp_port);
}
for &port in &ack.udp_ports {
if port != 0 && !ports.contains(&port) {
ports.push(port);
}
}
ports
}
fn receiver_udp_addr_for_socket(
peer: SocketAddr,
udp_ports: &[u16],
socket_index: usize,
) -> Result<SocketAddr, RqError> {
if udp_ports.is_empty() {
return Err(RqError::Io(std::io::Error::new(
std::io::ErrorKind::InvalidData,
"RQ receiver did not advertise any UDP data ports",
)));
}
Ok(SocketAddr::new(
peer.ip(),
udp_ports[socket_index % udp_ports.len()],
))
}
/// One logical file packed into a combined RaptorQ object (E-15 coalescing).
///
/// When [`ManifestEntry::members`] is non-empty the entry's content is the byte
/// concatenation of its members in `offset` order; the receiver splits the decoded
/// object back into the member files on commit. This amortizes the per-object
/// runtime overhead (decode pipeline / tasks / commit) that makes many-small-file
/// trees slow (profiled: ~81% runtime sync, 5.8× a same-byte single file).
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct PackedMember {
/// Path of this file relative to the transfer root.
pub rel_path: String,
/// Byte offset of this file within the combined object content.
pub offset: u64,
/// Byte length of this file.
pub len: u64,
/// Lowercase hex SHA-256 of this file's content (per-member integrity check).
pub sha256_hex: String,
}
/// One ordered RaptorQ object shard of a larger logical file.
///
/// A fragmented entry's manifest `rel_path` names the encoded object, while
/// this metadata names the logical file that will be reassembled and committed
/// after all shards verify. `sha256_hex` is the whole logical file SHA-256, not
/// the per-shard object hash (that remains [`ManifestEntry::sha256_hex`]).
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct LargeObjectFragment {
/// Logical file path relative to the transfer root.
pub rel_path: String,
/// Zero-based shard ordinal within the logical file.
pub shard_index: u32,
/// Total shard count for this logical file.
pub shard_count: u32,
/// Byte offset of this shard in the logical file.
pub logical_offset: u64,
/// Byte length carried by this shard.
pub len: u64,
/// Whole logical file size.
pub logical_size: u64,
/// Lowercase hex SHA-256 of the whole logical file.
pub sha256_hex: String,
}
/// One file within a transfer manifest.
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct ManifestEntry {
/// Stable index within the transfer (manifest order).
pub index: u32,
/// Path relative to the transfer root.
pub rel_path: String,
/// Entry size in bytes.
pub size: u64,
/// Lowercase hex SHA-256 of the entry content.
pub sha256_hex: String,
/// Files packed into this entry (E-15 coalescing). Empty = a normal single-file
/// entry whose content IS the file (prior wire format, byte-identical). Non-empty
/// = this entry is a combined object and these members are extracted by offset on
/// receive. `skip_serializing_if` keeps the no-packing wire byte-identical.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub members: Vec<PackedMember>,
/// Large-file multi-object metadata. Present when this manifest entry is one
/// ordered shard of a single logical file.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub fragment: Option<LargeObjectFragment>,
}
/// Transfer manifest carried in the `ObjectManifest` frame.
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct TransferManifest {
/// Stable transfer identifier (hex).
pub transfer_id: String,
/// Name of the transfer root (file name or directory name).
pub root_name: String,
/// Whether the root is a directory (vs a single file).
pub is_directory: bool,
/// Total bytes across all entries.
pub total_bytes: u64,
/// Lowercase hex of `MerkleRoot::from_graph` over the flat object graph.
pub merkle_root_hex: String,
/// File entries in manifest order.
pub entries: Vec<ManifestEntry>,
}
/// Object-level digest carried in the source-stream `ObjectComplete` trailer.
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
struct ObjectCompleteEntryDigest {
/// Manifest entry index.
index: u32,
/// Entry byte length observed by the sender while streaming.
size: u64,
/// Lowercase hex SHA-256 of this encoded object.
sha256_hex: String,
}
/// Logical-file digest carried in the source-stream `ObjectComplete` trailer.
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
struct ObjectCompleteLogicalDigest {
/// Logical path relative to the transfer root.
rel_path: String,
/// Logical file byte length observed by the sender while streaming.
size: u64,
/// Lowercase hex SHA-256 of the logical file.
sha256_hex: String,
}
/// Sender -> receiver marker for one finished spray/source-stream round.
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
struct RqRoundComplete {
/// Number of datagrams the sender emitted in the completed spray round.
///
/// The receiver uses this with its observed datagram count to report an
/// explicit loss fraction in `NeedMore`. Empty legacy `ObjectComplete`
/// frames are treated as unknown and fall back to sender-side inference.
#[serde(default)]
round_symbols_sent: u64,
/// Source-stream trailer digests for each encoded object. Empty on the UDP
/// RaptorQ datagram path, where manifest hashes remain authoritative.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
entry_digests: Vec<ObjectCompleteEntryDigest>,
/// Source-stream trailer digests for logical files whose digest is not
/// authoritatively carried by the manifest.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
logical_digests: Vec<ObjectCompleteLogicalDigest>,
/// Source-stream logical merkle root. Empty on the UDP datagram path, where
/// the manifest merkle root remains authoritative.
#[serde(default, skip_serializing_if = "Option::is_none")]
merkle_root_hex: Option<String>,
}
/// Sender-side UDP batch acceleration counters for the ATP-RQ symbol plane.
#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize, PartialEq, Eq)]
pub struct UdpSendAccelerationReport {
/// Logical per-socket batch flushes issued by the RQ sender.
pub flushes: u64,
/// Symbol datagrams reported as processed by the UDP layer.
pub datagrams: u64,
/// Payload bytes reported as processed by the UDP layer.
pub payload_bytes: u64,
/// Flushes that used an OS-native send batching syscall.
pub native_batch_flushes: u64,
/// Datagrams processed by flushes that used an OS-native send batching syscall.
pub native_batch_datagrams: u64,
/// Flushes that used UDP Generic Segmentation Offload.
pub gso_flushes: u64,
/// Datagrams processed by flushes that used UDP Generic Segmentation Offload.
pub gso_datagrams: u64,
/// Flushes that used the portable fallback loop for at least part of the batch.
pub fallback_flushes: u64,
/// Datagrams processed by flushes that used the portable fallback loop.
pub fallback_datagrams: u64,
/// Flushes that returned fewer datagrams than the sender queued.
pub partial_flushes: u64,
/// Flushes that surfaced a UDP-layer error string.
pub error_flushes: u64,
}
impl UdpSendAccelerationReport {
fn observe_flush_report(&mut self, report: RqSendBatchFlushReport) {
self.flushes = self
.flushes
.saturating_add(u64::try_from(report.socket_flushes).unwrap_or(u64::MAX));
self.datagrams = self
.datagrams
.saturating_add(u64::try_from(report.packets_processed).unwrap_or(u64::MAX));
self.payload_bytes = self
.payload_bytes
.saturating_add(u64::try_from(report.bytes_processed).unwrap_or(u64::MAX));
self.native_batch_flushes = self
.native_batch_flushes
.saturating_add(u64::try_from(report.native_flushes).unwrap_or(u64::MAX));
self.native_batch_datagrams = self
.native_batch_datagrams
.saturating_add(u64::try_from(report.native_packets).unwrap_or(u64::MAX));
self.gso_flushes = self
.gso_flushes
.saturating_add(u64::try_from(report.gso_flushes).unwrap_or(u64::MAX));
self.gso_datagrams = self
.gso_datagrams
.saturating_add(u64::try_from(report.gso_packets).unwrap_or(u64::MAX));
self.fallback_flushes = self
.fallback_flushes
.saturating_add(u64::try_from(report.fallback_flushes).unwrap_or(u64::MAX));
self.fallback_datagrams = self
.fallback_datagrams
.saturating_add(u64::try_from(report.fallback_packets).unwrap_or(u64::MAX));
self.partial_flushes = self
.partial_flushes
.saturating_add(u64::try_from(report.partial_flushes).unwrap_or(u64::MAX));
self.error_flushes = self
.error_flushes
.saturating_add(u64::try_from(report.error_flushes).unwrap_or(u64::MAX));
}
}
/// Receiver → sender fountain feedback: entries still needing more symbols.
#[derive(Debug, Clone, Serialize, Deserialize)]
struct NeedMore {
/// Entry indices that have not yet decoded.
pending: Vec<u32>,
/// Sparse systematic source symbols missing from incomplete blocks.
#[serde(default)]
source_symbols: Vec<SourceSymbolRequest>,
/// Matching RQ symbols observed by the receiver in the completed spray round.
///
/// This is the pacing/loss signal: duplicates and symbols that fail to
/// advance decode rank still prove the datagram arrived on the wire.
#[serde(default, skip_serializing_if = "Option::is_none")]
round_symbols_observed: Option<u64>,
/// Receiver-computed symbol loss fraction for the completed spray round.
///
/// AIMD uses this explicit wire-loss signal; pending decode pressure remains
/// separate and only feeds repair/FEC sizing.
#[serde(default, skip_serializing_if = "Option::is_none")]
round_loss_fraction: Option<f64>,
/// Matching RQ symbols accepted into a decoder in the completed spray round.
///
/// This remains diagnostic only; accepted symbols can stall on duplicate or
/// dependent repair rows and must not be treated as packet loss.
#[serde(default, skip_serializing_if = "Option::is_none")]
round_symbols_accepted: Option<u64>,
/// Aggregate decoder rank across the still-pending entries after this round.
///
/// Unlike `round_symbols_observed`, this is confirmed useful progress. The
/// sender uses rank deltas as a delivery-clocked congestion signal for lossy
/// cells where kernel overflow can make arrival loss appear artificially low.
#[serde(default, skip_serializing_if = "Option::is_none")]
pending_rank: Option<u64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pending_rank_columns: Option<u64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pending_rank_deficit: Option<u64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pending_decode_jobs: Option<u64>,
}
/// Request for retransmission of one systematic source symbol.
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
struct SourceSymbolRequest {
entry: u32,
sbn: u8,
esi: u32,
}
/// Receipt returned by the receiver in the `Proof` frame.
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct ReceiveReceipt {
/// Whether the receiver atomically committed the transfer to its destination.
pub committed: bool,
/// Total bytes received.
pub bytes_received: u64,
/// Number of files received.
pub files: u32,
/// Whether every entry's SHA-256 matched the manifest.
pub sha_ok: bool,
/// Whether the rebuilt merkle root matched the manifest.
pub merkle_ok: bool,
/// Total symbol datagrams the receiver accepted.
pub symbols_accepted: u64,
/// Fountain feedback rounds used.
pub feedback_rounds: u32,
/// Failure reason when `committed` is false.
#[serde(skip_serializing_if = "Option::is_none")]
pub reason: Option<String>,
/// Absolute destination paths that were committed.
pub committed_paths: Vec<String>,
}
/// Outcome of a successful [`send_path`] call.
#[derive(Debug, Clone)]
pub struct SendReport {
/// Transfer identifier.
pub transfer_id: String,
/// Total bytes sent.
pub bytes_sent: u64,
/// Number of files sent.
pub files: u32,
/// Total symbol datagrams emitted (across all feedback rounds).
pub symbols_sent: u64,
/// Fountain feedback rounds used.
pub feedback_rounds: u32,
/// Merkle root (hex) of the transfer.
pub merkle_root_hex: String,
/// The receiver's receipt.
pub receipt: ReceiveReceipt,
/// UDP send acceleration counters observed on the RQ symbol plane.
pub udp_send_acceleration: UdpSendAccelerationReport,
/// Peer control-plane address.
pub peer: SocketAddr,
}
/// Outcome of a successful received transfer.
#[derive(Debug, Clone)]
pub struct ReceiveReport {
/// Transfer identifier.
pub transfer_id: String,
/// Total bytes received.
pub bytes_received: u64,
/// Number of files committed.
pub files: u32,
/// Whether the transfer was committed to the destination.
pub committed: bool,
/// Total symbol datagrams accepted.
pub symbols_accepted: u64,
/// Fountain feedback rounds used.
pub feedback_rounds: u32,
/// Absolute committed paths.
pub committed_paths: Vec<PathBuf>,
/// Peer control-plane address.
pub peer: SocketAddr,
}
/// Outcome of a bonded donor's one-way symbol spray.
///
/// This is the B1 donor-side data-path report: the donor has proven its local
/// bytes match the agreed descriptor, materialized only its assigned ESIs, and
/// sent those symbols to the receiver endpoint(s). Receiver feedback and
/// `NeedMore`/`Close` handling are intentionally left to the B3 control loop.
#[derive(Debug, Clone)]
pub struct BondedDonorSendReport {
/// Stable transfer identifier from the bonded descriptor.
pub transfer_id: String,
/// Donor index used by the active assignment.
pub donor_index: u32,
/// Total donor count in the active assignment.
pub donor_count: u32,
/// Receiver UDP endpoints this donor connected to.
pub receiver_endpoints: Vec<SocketAddr>,
/// Descriptor entries considered for spray.
pub entries: usize,
/// Source blocks considered for spray.
pub blocks: usize,
/// Systematic/source symbols emitted.
pub source_symbols_sent: u64,
/// Repair/FEC symbols emitted.
pub repair_symbols_sent: u64,
/// Total UDP datagrams emitted by the donor.
pub symbols_sent: u64,
/// Pacing decision used for the donor's initial source-first spray.
pub pacing: BondedDonorPacingReport,
/// UDP send acceleration counters observed while spraying.
pub udp_send_acceleration: UdpSendAccelerationReport,
}
/// Pacing evidence for a bonded donor's source-first spray.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct BondedDonorPacingReport {
/// Initial token-bucket rate before any clean-link round-0 ramp.
pub initial_rate_bytes_per_sec: u64,
/// Final token-bucket rate after the donor spray completed.
pub final_rate_bytes_per_sec: u64,
/// Maximum symbols allowed in one pacer burst.
pub burst_symbols: u32,
/// Maximum bytes allowed in one pacer burst.
pub burst_bytes: u64,
/// Estimated authenticated symbol datagram size.
pub datagram_bytes: u32,
/// Whether the clean-link additive round-0 ramp is enabled.
pub clean_round0_ramp_enabled: bool,
}
#[derive(Debug, Clone, Copy)]
struct BondedDonorPacingDecision {
pacing: RqSprayPacing,
report: BondedDonorPacingReport,
}
fn bonded_donor_round0_pacing_decision(
transfer_id: &str,
config: &RqConfig,
fanout: usize,
) -> BondedDonorPacingDecision {
let mut state = RqAdaptiveSendState::new(transfer_tag(transfer_id), config, fanout);
let pacing = state.round0_tuning(config).pacing;
BondedDonorPacingDecision {
pacing,
report: BondedDonorPacingReport {
initial_rate_bytes_per_sec: pacing.rate_bytes_per_sec(),
final_rate_bytes_per_sec: pacing.rate_bytes_per_sec(),
burst_symbols: pacing.max_burst_size,
burst_bytes: pacing.burst_bytes(),
datagram_bytes: pacing.datagram_bytes,
clean_round0_ramp_enabled: round0_clean_ramp_enabled(config, pacing),
},
}
}
/// Donate this host's assigned slice of an agreed bonded RaptorQ fountain.
///
/// The descriptor is the receiver/donor agreement from Phase A. `source_root`
/// is this donor's local directory containing the descriptor's relative entry
/// paths; the descriptor proof rejects mismatched or escaping paths before any
/// symbol is sent. `receiver_endpoint` is the primary UDP endpoint selected by
/// the control plane, and any additional endpoints in `assignment` are used for
/// send fanout.
///
/// This B1 entrypoint performs the initial source-first spray only. It does not
/// read receiver feedback, honor `NeedMore`, or close the bonded control plane;
/// those belong to the B3 donor-control loop.
pub async fn donate_path(
cx: &Cx,
descriptor: &BondTransferDescriptor,
assignment: &DonorAssignment,
receiver_endpoint: SocketAddr,
source_root: &Path,
mut config: RqConfig,
) -> Result<BondedDonorSendReport, RqError> {
cx.checkpoint().map_err(|_| RqError::Cancelled)?;
assignment
.validate()
.map_err(|err| RqError::Coding(format!("invalid bonded donor assignment: {err}")))?;
descriptor
.prove_local_holding(source_root)
.await
.map_err(|err| RqError::Source(format!("bonded donor byte proof failed: {err}")))?;
apply_bonded_descriptor_config(descriptor, &mut config)?;
let symbol_auth = config.symbol_auth_context()?;
if assignment.requires_symbol_auth() && symbol_auth.is_none() {
return Err(RqError::Authentication(
"bonded donor assignment requires symbol auth but RqConfig allowed unauthenticated symbols"
.to_string(),
));
}
let repair_symbols_per_block = bonded_initial_repair_symbols_per_block(&config)?;
let schedule = schedule_bonded_donor_spray(descriptor, assignment, repair_symbols_per_block)
.map_err(|err| RqError::Coding(format!("bonded donor spray schedule failed: {err}")))?;
let receiver_endpoints = bonded_receiver_endpoints(assignment, receiver_endpoint);
let local_unspec = if receiver_endpoint.ip().is_ipv4() {
std::net::IpAddr::V4(std::net::Ipv4Addr::UNSPECIFIED)
} else {
std::net::IpAddr::V6(std::net::Ipv6Addr::UNSPECIFIED)
};
let mut sockets = Vec::with_capacity(receiver_endpoints.len());
for endpoint in &receiver_endpoints {
let sock = UdpSocket::bind(SocketAddr::new(local_unspec, 0)).await?;
sock.connect(*endpoint).await?;
let _ = sock.tune_buffers(UdpBufferConfig {
send_buffer_bytes: Some(16 * 1024 * 1024),
recv_buffer_bytes: None,
});
sockets.push(sock);
}
let pacing_decision =
bonded_donor_round0_pacing_decision(&descriptor.transfer_id, &config, sockets.len());
let mut pacing_report = pacing_decision.report;
let mut pacer = RqSprayPacer::new_round0(pacing_decision.pacing, &config, false);
let mut send_batch = RqPendingSendBatch::new(sockets.len());
let mut symbols_sent = 0u64;
let mut source_symbols_sent = 0u64;
let mut repair_symbols_sent = 0u64;
let mut rr = 0usize;
let mut dropper = 0u32;
let mut udp_send_acceleration = UdpSendAccelerationReport::default();
let tag = transfer_tag(&descriptor.transfer_id);
let assignment_mode = if assignment.esi_windows.is_empty() {
"static-residue"
} else {
"windowed"
};
bondtrace!(
"donor: spray_start transfer_id={} donor_index={} donor_count={} assignment_mode={} esi_windows={:?} receiver_endpoints={} blocks={} repair_symbols_per_block={} pacing_rate_Bps={} burst_symbols={} burst_bytes={} clean_round0_ramp={}",
descriptor.transfer_id,
assignment.donor_index,
assignment.donor_count,
assignment_mode,
assignment.esi_windows,
receiver_endpoints.len(),
schedule.blocks.len(),
repair_symbols_per_block,
pacing_report.initial_rate_bytes_per_sec,
pacing_report.burst_symbols,
pacing_report.burst_bytes,
pacing_report.clean_round0_ramp_enabled,
);
for block in &schedule.blocks {
cx.checkpoint().map_err(|_| RqError::Cancelled)?;
let entry = descriptor
.entry_by_index(block.geometry.entry_index)
.ok_or_else(|| {
RqError::Coding(format!(
"bonded donor schedule references unknown entry {}",
block.geometry.entry_index
))
})?;
let entry_path = bonded_donor_entry_path(source_root, &entry.rel_path)?;
let block_start =
usize::try_from(block.geometry.block_start).map_err(|_| RqError::TooLarge {
size: block.geometry.block_start,
max: u64::try_from(usize::MAX).unwrap_or(u64::MAX),
})?;
let block_len =
usize::try_from(block.geometry.block_bytes).map_err(|_| RqError::TooLarge {
size: block.geometry.block_bytes,
max: u64::try_from(usize::MAX).unwrap_or(u64::MAX),
})?;
bondtrace!(
"donor: block_start transfer_id={} donor_index={} entry={} sbn={} block_start={} block_bytes={} source_esis={} repair_esis={} stagger_delay_slots={} symbols_sent_so_far={} pacing_rate_Bps={}",
descriptor.transfer_id,
assignment.donor_index,
block.geometry.entry_index,
block.geometry.source_block_number,
block.geometry.block_start,
block.geometry.block_bytes,
block.source_esis.len(),
block.repair_esis.len(),
block.stagger_delay_slots,
symbols_sent,
pacer.pacing().rate_bytes_per_sec(),
);
let block_bytes = read_source_range(&entry_path, block_start, block_len).await?;
for emission in block.iter_symbol_emissions(schedule.donor_index) {
let symbol = encode_bonded_donor_emission(emission, &block_bytes, &config)?;
match emission.symbol_kind() {
SymbolKind::Source => source_symbols_sent = source_symbols_sent.saturating_add(1),
SymbolKind::Repair => repair_symbols_sent = repair_symbols_sent.saturating_add(1),
}
queue_bonded_donor_datagram(
cx,
&mut sockets,
&mut rr,
&mut symbols_sent,
&mut dropper,
tag,
block.geometry.entry_index,
&symbol,
&config,
&mut pacer,
symbol_auth.as_ref(),
&mut send_batch,
&mut udp_send_acceleration,
)
.await?;
}
bondtrace!(
"donor: block_done transfer_id={} donor_index={} entry={} sbn={} symbols_sent={} source_symbols_sent={} repair_symbols_sent={} pacing_rate_Bps={}",
descriptor.transfer_id,
assignment.donor_index,
block.geometry.entry_index,
block.geometry.source_block_number,
symbols_sent,
source_symbols_sent,
repair_symbols_sent,
pacer.pacing().rate_bytes_per_sec(),
);
}
let report = send_batch.flush(&mut sockets, &mut symbols_sent).await?;
udp_send_acceleration.observe_flush_report(report);
pacing_report.final_rate_bytes_per_sec = pacer.pacing().rate_bytes_per_sec();
bondtrace!(
"donor: spray_done transfer_id={} donor_index={} donor_count={} symbols_sent={} source_symbols_sent={} repair_symbols_sent={} initial_pacing_rate_Bps={} final_pacing_rate_Bps={} udp_flushes={} native_batch_datagrams={} gso_datagrams={} fallback_datagrams={}",
descriptor.transfer_id,
assignment.donor_index,
assignment.donor_count,
symbols_sent,
source_symbols_sent,
repair_symbols_sent,
pacing_report.initial_rate_bytes_per_sec,
pacing_report.final_rate_bytes_per_sec,
udp_send_acceleration.flushes,
udp_send_acceleration.native_batch_datagrams,
udp_send_acceleration.gso_datagrams,
udp_send_acceleration.fallback_datagrams,
);
Ok(BondedDonorSendReport {
transfer_id: descriptor.transfer_id.clone(),
donor_index: assignment.donor_index,
donor_count: assignment.donor_count,
receiver_endpoints,
entries: descriptor.entries.len(),
blocks: schedule.blocks.len(),
source_symbols_sent,
repair_symbols_sent,
symbols_sent,
pacing: pacing_report,
udp_send_acceleration,
})
}
// ─── Frame transport over the TCP control stream ─────────────────────────────
struct FrameTransport<S> {
stream: S,
codec: AtpFrameCodec,
rbuf: BytesMut,
// A control frame that the spray-time drain (`service_rq_spray_control`) pulled but that is
// NOT a KeepAlive — i.e. the receiver raced ahead and sent a terminal/feedback frame (Proof /
// ObjectRequest) while the sender was still spraying. We stash it here instead of erroring so
// the post-spray feedback loop's `recv()` returns it normally (fixes zz35zq: the fast-transfer
// "unexpected frame: got Proof, expected KeepAlive while spraying" abort).
stashed: Option<Frame>,
}
impl<S> FrameTransport<S>
where
S: AsyncReadExt + AsyncWriteExt + Unpin,
{
fn new(stream: S) -> Self {
Self {
stream,
codec: AtpFrameCodec::new(),
rbuf: BytesMut::new(),
stashed: None,
}
}
async fn send(&mut self, frame: &Frame) -> Result<(), RqError> {
self.send_unflushed(frame).await?;
self.flush().await
}
async fn send_unflushed(&mut self, frame: &Frame) -> Result<usize, RqError> {
let bytes = frame
.to_wire_bytes()
.map_err(|e| RqError::Frame(e.to_string()))?;
let len = bytes.len();
self.stream.write_all(&bytes).await?;
Ok(len)
}
async fn send_control_source_data_unflushed(
&mut self,
transfer_id: &str,
entry: u32,
offset: u64,
data: &[u8],
symbol_auth: Option<&SecurityContext>,
) -> Result<usize, RqError> {
let bytes = control_source_data_wire_frame(transfer_id, entry, offset, data, symbol_auth)?;
let len = bytes.len();
self.stream.write_all(&bytes).await?;
Ok(len)
}
async fn flush(&mut self) -> Result<(), RqError> {
self.stream.flush().await?;
Ok(())
}
async fn recv(&mut self) -> Result<Frame, RqError> {
// A frame deferred by the spray-time drain takes precedence (see `stashed`).
if let Some(frame) = self.stashed.take() {
return Ok(frame);
}
loop {
if let Some(frame) = self
.codec
.decode(&mut self.rbuf)
.map_err(|e| RqError::Frame(e.to_string()))?
{
return Ok(frame);
}
let mut tmp = vec![0u8; 65536];
let n = self.stream.read(&mut tmp).await?;
if n == 0 {
return Err(RqError::Io(std::io::Error::new(
std::io::ErrorKind::UnexpectedEof,
"peer closed control connection mid-transfer",
)));
}
self.rbuf.extend_from_slice(&tmp[..n]);
}
}
async fn try_recv_ready(&mut self) -> Result<Option<Frame>, RqError> {
use std::future::poll_fn;
use std::pin::Pin;
use std::task::Poll;
if self.stashed.is_some() {
return Ok(None);
}
if let Some(frame) = self
.codec
.decode(&mut self.rbuf)
.map_err(|e| RqError::Frame(e.to_string()))?
{
return Ok(Some(frame));
}
let mut tmp = [0u8; 4096];
let ready = poll_fn(|task_cx| {
let mut read_buf = ReadBuf::new(&mut tmp);
match Pin::new(&mut self.stream).poll_read(task_cx, &mut read_buf) {
Poll::Ready(Ok(())) => Poll::Ready(Ok(Some(read_buf.filled().len()))),
Poll::Ready(Err(err)) => Poll::Ready(Err(err)),
Poll::Pending => Poll::Ready(Ok(None)),
}
})
.await?;
let Some(n) = ready else {
return Ok(None);
};
if n == 0 {
return Err(RqError::Io(std::io::Error::new(
std::io::ErrorKind::UnexpectedEof,
"peer closed control connection mid-transfer",
)));
}
self.rbuf.extend_from_slice(&tmp[..n]);
self.codec
.decode(&mut self.rbuf)
.map_err(|e| RqError::Frame(e.to_string()))
}
}
async fn drain_sender_close_after_proof<S>(cx: &Cx, control: &mut FrameTransport<S>, phase: &str)
where
S: AsyncReadExt + AsyncWriteExt + Unpin,
{
if cx.is_cancel_requested() {
return;
}
for _ in 0..4 {
let frame = match control.try_recv_ready().await {
Ok(Some(frame)) => frame,
Ok(None) => break,
Err(err) => {
rqtrace!("receiver: sender Close after Proof unavailable phase={phase}: {err}");
return;
}
};
match frame.frame_type() {
FrameType::Close => {
rqtrace!("receiver: drained sender Close after Proof phase={phase}");
return;
}
FrameType::ObjectComplete | FrameType::KeepAlive => {
rqtrace!(
"receiver: draining late {:?} after Proof phase={phase}",
frame.frame_type()
);
}
other => {
rqtrace!("receiver: ignoring post-Proof frame {other:?} phase={phase}");
return;
}
}
}
rqtrace!("receiver: no ready sender Close after Proof phase={phase}");
}
fn tune_control_stream_for_bulk_source(stream: &TcpStream) {
let _ = stream.set_nodelay(true);
#[cfg(not(target_arch = "wasm32"))]
if let Some(std_stream) = stream.try_as_std() {
let sock = socket2::SockRef::from(std_stream);
let _ = sock.set_send_buffer_size(RQ_CONTROL_STREAM_SOCKET_BUFFER_BYTES);
let _ = sock.set_recv_buffer_size(RQ_CONTROL_STREAM_SOCKET_BUFFER_BYTES);
}
}
// ─── Helpers (entry walk + merkle, shared definition with transport_tcp) ─────
fn json_frame<T: Serialize>(ty: FrameType, value: &T) -> Result<Frame, RqError> {
let payload = serde_json::to_vec(value).map_err(|e| RqError::Control(e.to_string()))?;
Frame::new(ProtocolVersion::CURRENT, ty, payload).map_err(|e| RqError::Frame(e.to_string()))
}
fn parse_json<T: for<'de> Deserialize<'de>>(frame: &Frame) -> Result<T, RqError> {
serde_json::from_slice(frame.payload()).map_err(|e| RqError::Control(e.to_string()))
}
fn control_source_data_chunk_bytes(auth_enabled: bool) -> usize {
if auth_enabled {
RQ_CONTROL_SOURCE_AUTH_CHUNK_BYTES
} else {
RQ_CONTROL_SOURCE_CHUNK_BYTES
}
}
fn control_source_data_auth_tag_bytes(auth_enabled: bool) -> usize {
if auth_enabled { TAG_SIZE } else { 0 }
}
fn control_source_data_auth_symbol(
transfer_id: &str,
entry: u32,
offset: u64,
data: &[u8],
) -> Symbol {
let entry_id = entry_object_id(transfer_id, entry);
let object_id = ObjectId::new(
entry_id.high() ^ 0xC057_DA7A_AA71_0001,
entry_id.low() ^ 0xA771_C057_5EED_0001,
);
let mut payload =
Vec::with_capacity(RQ_CONTROL_SOURCE_AUTH_SYMBOL_DOMAIN.len() + 8 + data.len());
payload.extend_from_slice(RQ_CONTROL_SOURCE_AUTH_SYMBOL_DOMAIN);
payload.extend_from_slice(&offset.to_be_bytes());
payload.extend_from_slice(data);
Symbol::new(SymbolId::new(object_id, 0, 0), payload, SymbolKind::Source)
}
fn sign_control_source_data_tag(
context: &SecurityContext,
transfer_id: &str,
entry: u32,
offset: u64,
data: &[u8],
) -> AuthenticationTag {
let symbol = control_source_data_auth_symbol(transfer_id, entry, offset, data);
context.sign_symbol_tag(&symbol)
}
fn verify_control_source_data_tag(
context: &SecurityContext,
transfer_id: &str,
entry: u32,
offset: u64,
data: &[u8],
tag: AuthenticationTag,
) -> Result<(), RqError> {
let symbol = control_source_data_auth_symbol(transfer_id, entry, offset, data);
let mut auth = AuthenticatedSymbol::from_parts(symbol, tag);
if context.verify_authenticated_symbol(&mut auth).is_ok() && auth.is_verified() {
Ok(())
} else {
Err(RqError::Authentication(format!(
"control source ObjectData authentication failed for entry {entry} offset {offset}"
)))
}
}
fn control_source_data_payload(
transfer_id: &str,
entry: u32,
offset: u64,
data: &[u8],
symbol_auth: Option<&SecurityContext>,
) -> Vec<u8> {
let auth_len = control_source_data_auth_tag_bytes(symbol_auth.is_some());
let mut payload = Vec::with_capacity(RQ_CONTROL_SOURCE_DATA_HEADER + auth_len + data.len());
payload.extend_from_slice(&entry.to_be_bytes());
payload.extend_from_slice(&offset.to_be_bytes());
if let Some(context) = symbol_auth {
let tag = sign_control_source_data_tag(context, transfer_id, entry, offset, data);
payload.extend_from_slice(tag.as_bytes());
}
payload.extend_from_slice(data);
payload
}
#[cfg(test)]
fn control_source_data_frame(entry: u32, offset: u64, data: &[u8]) -> Result<Frame, RqError> {
control_source_data_frame_with_auth("control-source-test", entry, offset, data, None)
}
#[cfg(test)]
fn control_source_data_frame_with_auth(
transfer_id: &str,
entry: u32,
offset: u64,
data: &[u8],
symbol_auth: Option<&SecurityContext>,
) -> Result<Frame, RqError> {
let payload = control_source_data_payload(transfer_id, entry, offset, data, symbol_auth);
Frame::new(ProtocolVersion::CURRENT, FrameType::ObjectData, payload)
.map_err(|e| RqError::Frame(e.to_string()))
}
fn control_source_data_wire_frame(
transfer_id: &str,
entry: u32,
offset: u64,
data: &[u8],
symbol_auth: Option<&SecurityContext>,
) -> Result<BytesMut, RqError> {
let auth_len = control_source_data_auth_tag_bytes(symbol_auth.is_some());
let payload_len = RQ_CONTROL_SOURCE_DATA_HEADER
.checked_add(auth_len)
.and_then(|len| len.checked_add(data.len()))
.ok_or_else(|| {
RqError::Frame("control source ObjectData payload length overflow".into())
})?;
let max_payload_len = RQ_CONTROL_SOURCE_DATA_HEADER
.checked_add(auth_len)
.and_then(|len| len.checked_add(control_source_data_chunk_bytes(symbol_auth.is_some())))
.ok_or_else(|| {
RqError::Frame("control source ObjectData payload length overflow".into())
})?;
if payload_len > max_payload_len {
return Err(RqError::Frame(format!(
"control source ObjectData payload too large: {payload_len} bytes (max {max_payload_len})"
)));
}
let payload_len_u64 = u64::try_from(payload_len)
.map_err(|_| RqError::Frame("control source ObjectData payload too large".into()))?;
let header_len = control_frame_header_len(
ProtocolVersion::CURRENT,
FrameType::ObjectData,
payload_len_u64,
)?;
let total_len = header_len
.checked_add(payload_len)
.ok_or_else(|| RqError::Frame("control source ObjectData frame length overflow".into()))?;
let max_frame_size = usize::try_from(MAX_FRAME_SIZE).unwrap_or(usize::MAX);
if total_len > max_frame_size {
return Err(RqError::Frame(format!(
"control source ObjectData frame too large: {total_len} bytes (max {max_frame_size})"
)));
}
let mut wire = BytesMut::with_capacity(total_len);
encode_control_frame_varint(&mut wire, u64::from(ProtocolVersion::CURRENT.0))?;
encode_control_frame_varint(&mut wire, FrameType::ObjectData as u64)?;
encode_control_frame_varint(&mut wire, payload_len_u64)?;
encode_control_frame_varint(&mut wire, 0)?;
wire.extend_from_slice(&control_source_data_payload(
transfer_id,
entry,
offset,
data,
symbol_auth,
));
debug_assert_eq!(wire.len(), total_len);
Ok(wire)
}
fn control_frame_header_len(
version: ProtocolVersion,
frame_type: FrameType,
payload_len: u64,
) -> Result<usize, RqError> {
let version_len = control_frame_varint(u64::from(version.0))?.encoded_len();
let type_len = frame_type.to_varint().encoded_len();
let payload_len_len = control_frame_varint(payload_len)?.encoded_len();
version_len
.checked_add(type_len)
.and_then(|len| len.checked_add(payload_len_len))
.and_then(|len| len.checked_add(1))
.ok_or_else(|| RqError::Frame("control frame header length overflow".into()))
}
fn control_frame_varint(value: u64) -> Result<VarInt, RqError> {
VarInt::try_from(value).map_err(|err| RqError::Frame(err.to_string()))
}
fn encode_control_frame_varint(dst: &mut BytesMut, value: u64) -> Result<(), RqError> {
control_frame_varint(value)?
.encode(dst)
.into_result()
.map_err(|err| RqError::Frame(err.to_string()))
}
struct ControlSourceData<'a> {
entry: u32,
offset: u64,
data: &'a [u8],
}
fn parse_control_source_data_frame<'a>(
frame: &'a Frame,
transfer_id: &str,
symbol_auth: Option<&SecurityContext>,
) -> Result<ControlSourceData<'a>, RqError> {
let payload = frame.payload();
if payload.len() < RQ_CONTROL_SOURCE_DATA_HEADER {
return Err(RqError::Frame(format!(
"ObjectData frame shorter than {RQ_CONTROL_SOURCE_DATA_HEADER}-byte source header"
)));
}
let entry = u32::from_be_bytes(payload[0..4].try_into().expect("entry header width"));
let offset = u64::from_be_bytes(payload[4..12].try_into().expect("offset header width"));
let data = if let Some(context) = symbol_auth {
if payload.len() < RQ_CONTROL_SOURCE_AUTH_DATA_HEADER {
return Err(RqError::Authentication(format!(
"authenticated ObjectData frame shorter than {RQ_CONTROL_SOURCE_AUTH_DATA_HEADER}-byte source auth header"
)));
}
let mut tag_bytes = [0u8; TAG_SIZE];
tag_bytes.copy_from_slice(
&payload[RQ_CONTROL_SOURCE_DATA_HEADER..RQ_CONTROL_SOURCE_AUTH_DATA_HEADER],
);
let data = &payload[RQ_CONTROL_SOURCE_AUTH_DATA_HEADER..];
verify_control_source_data_tag(
context,
transfer_id,
entry,
offset,
data,
AuthenticationTag::from_bytes(tag_bytes),
)?;
data
} else {
&payload[RQ_CONTROL_SOURCE_DATA_HEADER..]
};
Ok(ControlSourceData {
entry,
offset,
data,
})
}
fn parse_round_complete(frame: &Frame) -> Result<RqRoundComplete, RqError> {
if frame.payload().is_empty() {
Ok(RqRoundComplete::default())
} else {
parse_json(frame)
}
}
fn receiver_round_loss_fraction(observed: u64, sent: u64) -> Option<f64> {
if sent == 0 {
return None;
}
let observed = observed.min(sent);
Some((1.0 - observed as f64 / sent as f64).clamp(0.0, 0.90))
}
fn parse_and_validate_manifest_frame(
frame: &Frame,
config: &RqConfig,
) -> Result<TransferManifest, RqError> {
let manifest: TransferManifest = parse_json(frame)?;
validate_manifest(&manifest, config)?;
Ok(manifest)
}
/// Derive the per-entry RaptorQ [`ObjectId`] deterministically from the transfer
/// id and entry index, so sender and receiver agree without extra signaling.
fn entry_object_id(transfer_id: &str, index: u32) -> ObjectId {
let mut hasher = Sha256::new();
hasher.update(b"asupersync.atp.rq.entry-object-id.v1\0");
hasher.update(transfer_id.as_bytes());
hasher.update(index.to_be_bytes());
let d = hasher.finalize();
let high = u64::from_be_bytes([d[0], d[1], d[2], d[3], d[4], d[5], d[6], d[7]]);
let low = u64::from_be_bytes([d[8], d[9], d[10], d[11], d[12], d[13], d[14], d[15]]);
ObjectId::new(high, low)
}
/// First 8 bytes of the transfer id hex, as a datagram-tag `u64` (cheap stray
/// packet filter — not a security boundary).
fn transfer_tag(transfer_id: &str) -> u64 {
let mut hasher = Sha256::new();
hasher.update(b"asupersync.atp.rq.tag.v1\0");
hasher.update(transfer_id.as_bytes());
let d = hasher.finalize();
u64::from_be_bytes([d[0], d[1], d[2], d[3], d[4], d[5], d[6], d[7]])
}
/// Hash-pass buffer size for sender manifest construction. This bounds the
/// manifest pass independently of transfer size.
const RQ_STREAM_HASH_BUFFER_SIZE: usize = 1024 * 1024;
#[derive(Debug, Clone)]
struct RqSourceEntry {
rel_path: String,
abs_path: PathBuf,
/// Byte offset in `abs_path` where this encoded object starts.
source_offset: u64,
/// Byte length of this encoded object. `None` means the whole file at
/// `abs_path`, preserving the historical no-split path.
source_len: Option<u64>,
/// Logical files packed into this combined RaptorQ object (E-15 coalescing).
/// Empty = a normal single-file entry whose content IS the file at `abs_path`
/// (prior behavior, byte-identical wire). Non-empty = `abs_path` points at a
/// temp file holding the concatenation of these members in `offset` order, and
/// the receiver splits it back into the member files on commit.
members: Vec<PackedMember>,
/// Large-file multi-object metadata for this encoded object.
fragment: Option<LargeObjectFragment>,
}
async fn collect_entries(root: &Path) -> Result<(String, bool, Vec<RqSourceEntry>), RqError> {
let meta = crate::fs::metadata(root)
.await
.map_err(|e| RqError::Source(format!("{}: {e}", root.display())))?;
let root_name = root.file_name().map_or_else(
|| "transfer".to_string(),
|n| n.to_string_lossy().into_owned(),
);
if meta.is_file() {
return Ok((
root_name.clone(),
false,
vec![RqSourceEntry {
rel_path: root_name,
abs_path: root.to_path_buf(),
source_offset: 0,
source_len: None,
members: Vec::new(),
fragment: None,
}],
));
}
if meta.is_dir() {
let mut entries = Vec::new();
collect_dir(root, String::new(), &mut entries).await?;
entries.sort_by(|a, b| a.rel_path.cmp(&b.rel_path));
return Ok((root_name, true, entries));
}
Err(RqError::Source(format!(
"{}: not a regular file or directory",
root.display()
)))
}
fn collect_dir<'a>(
dir: &'a Path,
prefix: String,
out: &'a mut Vec<RqSourceEntry>,
) -> std::pin::Pin<Box<dyn std::future::Future<Output = Result<(), RqError>> + Send + 'a>> {
Box::pin(async move {
let mut read_dir = crate::fs::read_dir(dir)
.await
.map_err(|e| RqError::Source(format!("{}: {e}", dir.display())))?;
let mut children: Vec<(String, PathBuf, bool)> = Vec::new();
while let Some(entry) = read_dir
.next_entry()
.await
.map_err(|e| RqError::Source(format!("{}: {e}", dir.display())))?
{
let name = entry.file_name().to_string_lossy().into_owned();
let path = entry.path();
let ft = entry
.file_type()
.await
.map_err(|e| RqError::Source(format!("{}: {e}", path.display())))?;
children.push((name, path, ft.is_dir()));
}
children.sort_by(|a, b| a.0.cmp(&b.0));
for (name, path, is_dir) in children {
let rel = if prefix.is_empty() {
name.clone()
} else {
format!("{prefix}/{name}")
};
if is_dir {
collect_dir(&path, rel, out).await?;
} else {
out.push(RqSourceEntry {
rel_path: rel,
abs_path: path,
source_offset: 0,
source_len: None,
members: Vec::new(),
fragment: None,
});
}
}
Ok(())
})
}
async fn source_entry_size(entry: &RqSourceEntry) -> Result<u64, RqError> {
if let Some(len) = entry.source_len {
return Ok(len);
}
crate::fs::metadata(&entry.abs_path)
.await
.map(|metadata| metadata.len())
.map_err(|e| RqError::Source(format!("{}: {e}", entry.abs_path.display())))
}
async fn source_entry_sizes(entries: &[RqSourceEntry]) -> Result<Vec<u64>, RqError> {
let mut sizes = Vec::with_capacity(entries.len());
for entry in entries {
sizes.push(source_entry_size(entry).await?);
}
Ok(sizes)
}
async fn source_entries_total_bytes(entries: &[RqSourceEntry]) -> Result<u64, RqError> {
let mut total = 0u64;
for entry in entries {
let size = source_entry_size(entry).await?;
total = total.checked_add(size).ok_or(RqError::TooLarge {
size: u64::MAX,
max: u64::MAX,
})?;
}
Ok(total)
}
/// E-15 tree coalescing (send side): greedily pack sub-threshold files into fewer,
/// larger combined RaptorQ objects.
///
/// `entries` arrives in manifest (sorted `rel_path`) order. Files whose size is
/// `< PACK_THRESHOLD` are binned greedily, in order, into packs that hold at most
/// `PACK_TARGET.min(max_object_size(config.max_block_size))` bytes (a pack always
/// holds at least one file). A pack of **two or more** files is materialized as a
/// temp file holding the byte concatenation of
/// its members in order; the resulting [`RqSourceEntry`] points at that temp file
/// and carries the [`PackedMember`] offset/len/sha table. A pack of exactly one
/// file (a lone leftover small file, or a single small file with no neighbor) is
/// emitted unchanged (no temp, empty `members`) so it stays byte-identical to the
/// non-packing wire. Files `>= PACK_THRESHOLD` are always emitted unchanged.
///
/// Returns `(new_entries, logical_digests, tempdir)` where `logical_digests` holds
/// one [`EntryDigest`] per **logical file** (members flattened) — the input to the
/// LOGICAL merkle root. For the no-packing case `logical_digests` equals the
/// per-file digests the caller would have computed itself, so the merkle root is
/// byte-identical to prior transfers. `tempdir` (if any) owns every materialized
/// pack temp file and MUST be kept alive until the spray loop has finished reading
/// them; dropping it removes the temp files.
///
/// # Errors
///
/// Returns [`RqError::Source`] if a source file cannot be hashed or a pack temp
/// file cannot be created/written.
async fn pack_small_files(
entries: Vec<RqSourceEntry>,
config: &RqConfig,
) -> Result<
(
Vec<RqSourceEntry>,
Vec<EntryDigest>,
Option<tempfile::TempDir>,
),
RqError,
> {
pack_small_files_with_deferred_singleton_digests(entries, config, false).await
}
async fn pack_small_files_with_deferred_singleton_digests(
entries: Vec<RqSourceEntry>,
config: &RqConfig,
defer_singleton_digests: bool,
) -> Result<
(
Vec<RqSourceEntry>,
Vec<EntryDigest>,
Option<tempfile::TempDir>,
),
RqError,
> {
let mut hash_buf = vec![0u8; RQ_STREAM_HASH_BUFFER_SIZE];
// Packed objects are intentionally not split by E-12, so a pack must stay
// inside the configured one-object SBN envelope. If a single small file is
// larger than this cap it remains unpacked and `split_large_entries` handles
// it as ranged objects.
let symbol_size = usize::from(config.symbol_size.max(1));
let pack_target = PACK_TARGET.min(
u64::try_from(max_object_size(config.max_block_size.max(symbol_size))).unwrap_or(u64::MAX),
);
// Group consecutive small files into packs. Each `Vec<usize>` is a list of
// indices into `entries` (the original sorted order is preserved so member
// offsets and the logical-digest order are deterministic).
let mut groups: Vec<Vec<usize>> = Vec::new();
let mut current: Vec<usize> = Vec::new();
let mut current_bytes: u64 = 0;
for (idx, entry) in entries.iter().enumerate() {
// Hash here purely to learn the size cheaply? No — hashing twice would
// double the disk read. Instead size is read from metadata; the content
// sha is computed once below (for packed members) or by the caller's
// per-object loop (for unpacked entries via the temp/real abs_path).
let size = crate::fs::metadata(&entry.abs_path)
.await
.map_err(|e| RqError::Source(format!("{}: {e}", entry.abs_path.display())))?
.len();
if size >= PACK_THRESHOLD {
// Flush any in-progress small-file group, then emit this large file
// as its own (unpacked) singleton group.
if !current.is_empty() {
groups.push(std::mem::take(&mut current));
current_bytes = 0;
}
groups.push(vec![idx]);
continue;
}
// Small file: would adding it overflow the current pack? Start a fresh
// pack if so (but never split a pack to empty — a single oversized-for-
// -target small file still forms its own pack).
if !current.is_empty() && current_bytes.saturating_add(size) > pack_target {
groups.push(std::mem::take(&mut current));
current_bytes = 0;
}
current.push(idx);
current_bytes = current_bytes.saturating_add(size);
}
if !current.is_empty() {
groups.push(current);
}
// If no group holds 2+ files, packing would do nothing useful. Return the
// entries unchanged and compute per-file logical digests (byte-identical to
// the caller's prior per-file digest pass).
let packs_anything = groups.iter().any(|g| g.len() >= 2);
if !packs_anything {
let mut logical_digests = Vec::with_capacity(entries.len());
if !defer_singleton_digests {
for entry in &entries {
let (size, content_id, content_sha256) =
hash_file_streaming(&entry.abs_path, &mut hash_buf)
.await
.map_err(|e| RqError::Source(e.into_message()))?;
logical_digests.push(EntryDigest {
rel_path: entry.rel_path.clone(),
size,
content_id,
content_sha256,
});
}
}
return Ok((entries, logical_digests, None));
}
let tempdir = tempfile::Builder::new()
.prefix(".atp-rq-pack-")
.tempdir()
.map_err(RqError::Io)?;
let mut new_entries: Vec<RqSourceEntry> = Vec::with_capacity(groups.len());
let mut logical_digests: Vec<EntryDigest> = Vec::with_capacity(entries.len());
for (pack_idx, group) in groups.iter().enumerate() {
if group.len() < 2 {
// Singleton (a lone small file or a >= threshold file): emit unchanged
// and push its own logical digest. Byte-identical to today.
let entry = &entries[group[0]];
if !defer_singleton_digests {
let (size, content_id, content_sha256) =
hash_file_streaming(&entry.abs_path, &mut hash_buf)
.await
.map_err(|e| RqError::Source(e.into_message()))?;
logical_digests.push(EntryDigest {
rel_path: entry.rel_path.clone(),
size,
content_id,
content_sha256,
});
}
new_entries.push(entry.clone());
continue;
}
// 2+ small files → materialize a combined object.
let pack_path = tempdir.path().join(format!("pack-{pack_idx}"));
let mut pack_file = crate::fs::File::create(&pack_path)
.await
.map_err(|e| RqError::Source(format!("{}: {e}", pack_path.display())))?;
let mut members: Vec<PackedMember> = Vec::with_capacity(group.len());
let mut offset: u64 = 0;
for &member_idx in group {
let entry = &entries[member_idx];
let (len, content_id, content_sha256) = append_file_to_pack_with_digest(
&entry.abs_path,
&pack_path,
&mut pack_file,
&mut hash_buf,
)
.await?;
members.push(PackedMember {
rel_path: entry.rel_path.clone(),
offset,
len,
sha256_hex: hex_encode(&content_sha256),
});
logical_digests.push(EntryDigest {
rel_path: entry.rel_path.clone(),
size: len,
content_id,
content_sha256,
});
offset = offset.saturating_add(len);
}
pack_file
.flush()
.await
.map_err(|e| RqError::Source(format!("{}: {e}", pack_path.display())))?;
drop(pack_file);
new_entries.push(RqSourceEntry {
rel_path: format!(".atp-pack-{pack_idx}"),
abs_path: pack_path,
source_offset: 0,
source_len: None,
members,
fragment: None,
});
}
Ok((new_entries, logical_digests, Some(tempdir)))
}
async fn append_file_to_pack_with_digest(
path: &Path,
pack_path: &Path,
pack_file: &mut crate::fs::File,
buf: &mut [u8],
) -> Result<(u64, crate::atp::object::ObjectId, [u8; 32]), RqError> {
let mut src = crate::fs::File::open(path)
.await
.map_err(|e| RqError::Source(format!("{}: {e}", path.display())))?;
let mut sha = Sha256::new();
let mut cid = crate::atp::object::ContentId::streaming();
let mut size = 0u64;
loop {
let n = src
.read(buf)
.await
.map_err(|e| RqError::Source(format!("{}: {e}", path.display())))?;
if n == 0 {
break;
}
let chunk = &buf[..n];
sha.update(chunk);
cid.update(chunk);
pack_file
.write_all(chunk)
.await
.map_err(|e| RqError::Source(format!("{}: {e}", pack_path.display())))?;
size = size.checked_add(n as u64).ok_or_else(|| {
RqError::Coding(format!("{}: packed member size overflow", path.display()))
})?;
}
Ok((
size,
crate::atp::object::ObjectId::content(cid.finalize()),
sha.finalize().into(),
))
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum ManifestDigestMode {
Manifest,
SourceStreamTrailer,
}
/// Split large unpacked entries into ordered RaptorQ objects while preserving the
/// logical file digest list used for the transfer merkle root.
async fn split_large_entries(
entries: Vec<RqSourceEntry>,
logical_digests: &[EntryDigest],
config: &RqConfig,
) -> Result<Vec<RqSourceEntry>, RqError> {
split_large_entries_with_digest_mode(
entries,
logical_digests,
config,
ManifestDigestMode::Manifest,
)
.await
}
async fn split_large_entries_with_digest_mode(
entries: Vec<RqSourceEntry>,
logical_digests: &[EntryDigest],
config: &RqConfig,
digest_mode: ManifestDigestMode,
) -> Result<Vec<RqSourceEntry>, RqError> {
let symbol_size = usize::from(config.symbol_size.max(1));
let block_size = config.max_block_size.max(symbol_size);
let split_config =
MultiObjectSplitConfig::new(u64::try_from(block_size).map_err(|_| {
RqError::Coding(format!("max_block_size does not fit u64: {block_size}"))
})?);
let mut out = Vec::with_capacity(entries.len());
for entry in entries {
if !entry.members.is_empty() {
out.push(entry);
continue;
}
let size = crate::fs::metadata(&entry.abs_path)
.await
.map_err(|e| RqError::Source(format!("{}: {e}", entry.abs_path.display())))?
.len();
let plan = plan_multi_object_split(size, split_config)
.map_err(|e| RqError::Coding(e.to_string()))?;
if !plan.is_split() {
out.push(entry);
continue;
}
let logical_digest = logical_digests
.iter()
.find(|digest| digest.rel_path == entry.rel_path);
if digest_mode == ManifestDigestMode::Manifest && logical_digest.is_none() {
return Err(RqError::Coding(format!(
"large-entry split missing logical digest for {}",
entry.rel_path
)));
}
let shard_count = u32::try_from(plan.shard_count()).map_err(|_| {
RqError::Coding(format!(
"large-entry split produced too many shards for {}",
entry.rel_path
))
})?;
let whole_sha256_hex = logical_digest
.map(|digest| hex_encode(&digest.content_sha256))
.unwrap_or_else(sha256_hex_placeholder);
for shard in plan.shards {
let object_rel_path = format!(".atp-fragment-{}-{}", out.len(), shard.shard_index);
out.push(RqSourceEntry {
rel_path: object_rel_path,
abs_path: entry.abs_path.clone(),
source_offset: shard.logical_offset,
source_len: Some(shard.len),
members: Vec::new(),
fragment: Some(LargeObjectFragment {
rel_path: entry.rel_path.clone(),
shard_index: shard.shard_index,
shard_count,
logical_offset: shard.logical_offset,
len: shard.len,
logical_size: plan.logical_size,
sha256_hex: whole_sha256_hex.clone(),
}),
});
}
}
Ok(out)
}
/// Reduce an attacker-controlled `root_name` to a single safe path component
/// joined under `dest_dir`.
///
/// `manifest.root_name` arrives off the wire, and `Path::join` *replaces* the
/// base when its argument is absolute, so `dest_dir.join(&root_name)` with an
/// absolute (or separator-bearing) `root_name` would escape the destination
/// directory entirely — `crate::fs::write_atomic` validates with
/// `allow_absolute = true`, so it would not catch an absolute target. Senders
/// already set `root_name` to a bare file name (see `collect_entries`), so
/// collapsing to the final path component is loss-free for legitimate
/// transfers while fully containing hostile ones (matches `transport_tcp`).
fn safe_base_for_root_name(dest_dir: &Path, root_name: &str) -> Result<PathBuf, RqError> {
if root_name.is_empty() {
return Err(RqError::Source("manifest root_name is empty".to_string()));
}
let component = Path::new(root_name)
.file_name()
.ok_or_else(|| RqError::Source(format!("unsafe manifest root_name: {root_name}")))?;
// `file_name()` never yields `.`/`..`/separators, but guard defensively
// in case of platform-specific surprises.
let component_str = component.to_string_lossy();
if component_str == "."
|| component_str == ".."
|| component_str.contains('/')
|| component_str.contains('\\')
{
return Err(RqError::Source(format!(
"unsafe manifest root_name: {root_name}"
)));
}
Ok(dest_dir.join(component))
}
/// Validate an incoming transfer manifest before allocating per-entry decoders.
///
/// The manifest is fully controlled by the peer. `total_bytes` alone is not a
/// sufficient memory bound because each entry size also drives RaptorQ decoder
/// metadata and each entry creates receiver bookkeeping.
fn validate_manifest(manifest: &TransferManifest, config: &RqConfig) -> Result<(), RqError> {
if manifest.transfer_id.is_empty()
|| manifest.transfer_id.len() > 64
|| !manifest
.transfer_id
.bytes()
.all(|b| b.is_ascii_alphanumeric())
{
return Err(RqError::Frame(format!(
"unsafe manifest transfer_id: {}",
manifest.transfer_id
)));
}
if manifest.total_bytes > config.max_transfer_bytes {
return Err(RqError::TooLarge {
size: manifest.total_bytes,
max: config.max_transfer_bytes,
});
}
validate_manifest_sha256_hex("manifest merkle_root_hex", &manifest.merkle_root_hex)?;
if manifest.entries.len() > MAX_MANIFEST_ENTRIES {
return Err(RqError::Frame(format!(
"manifest declares {} entries (max {MAX_MANIFEST_ENTRIES})",
manifest.entries.len()
)));
}
let single_file_fragmented = !manifest.is_directory
&& !manifest.entries.is_empty()
&& manifest
.entries
.iter()
.all(|entry| entry.fragment.is_some());
if !manifest.is_directory && manifest.entries.len() != 1 && !single_file_fragmented {
return Err(RqError::Frame(format!(
"single-file transfer manifest declares {} entries",
manifest.entries.len()
)));
}
#[derive(Debug)]
struct FragmentGroupValidation {
logical_size: u64,
shard_count: u32,
sha256_hex: String,
shards: Vec<(u32, u64, u64)>,
}
let mut seen_object_rel_paths: BTreeSet<String> = BTreeSet::new();
let mut seen_logical_rel_paths: BTreeSet<String> = BTreeSet::new();
let mut fragment_groups: BTreeMap<String, FragmentGroupValidation> = BTreeMap::new();
let declared_total =
manifest
.entries
.iter()
.enumerate()
.try_fold(0u64, |acc, (position, entry)| {
let expected = u32::try_from(position).map_err(|_| {
RqError::Frame("manifest contains too many indexed entries".to_string())
})?;
if entry.index != expected {
return Err(RqError::Frame(format!(
"manifest entry index {} does not match position {expected}",
entry.index
)));
}
validate_manifest_rel_path(&entry.rel_path)?;
if !seen_object_rel_paths.insert(entry.rel_path.clone()) {
return Err(RqError::Frame(format!(
"duplicate manifest rel_path: {}",
entry.rel_path
)));
}
validate_manifest_sha256_hex("manifest entry sha256_hex", &entry.sha256_hex)?;
if let Some(fragment) = &entry.fragment {
if !entry.members.is_empty() {
return Err(RqError::Frame(format!(
"manifest entry {} cannot be both packed and fragmented",
entry.rel_path
)));
}
validate_manifest_rel_path(&fragment.rel_path)?;
validate_manifest_sha256_hex(
"manifest fragment sha256_hex",
&fragment.sha256_hex,
)?;
if fragment.shard_count == 0 || fragment.shard_index >= fragment.shard_count {
return Err(RqError::Frame(format!(
"fragment {} has invalid shard {}/{}",
fragment.rel_path, fragment.shard_index, fragment.shard_count
)));
}
if fragment.len != entry.size {
return Err(RqError::Frame(format!(
"fragment {} len {} does not match object {} size {}",
fragment.rel_path, fragment.len, entry.rel_path, entry.size
)));
}
let end = fragment
.logical_offset
.checked_add(fragment.len)
.ok_or_else(|| {
RqError::Frame(format!(
"fragment {} byte range overflows",
fragment.rel_path
))
})?;
if end > fragment.logical_size {
return Err(RqError::Frame(format!(
"fragment {} range ends at {end} beyond logical size {}",
fragment.rel_path, fragment.logical_size
)));
}
let group = fragment_groups.entry(fragment.rel_path.clone()).or_insert_with(
|| FragmentGroupValidation {
logical_size: fragment.logical_size,
shard_count: fragment.shard_count,
sha256_hex: fragment.sha256_hex.clone(),
shards: Vec::new(),
},
);
if group.logical_size != fragment.logical_size
|| group.shard_count != fragment.shard_count
|| group.sha256_hex != fragment.sha256_hex
{
return Err(RqError::Frame(format!(
"fragment {} metadata is inconsistent across shards",
fragment.rel_path
)));
}
group
.shards
.push((fragment.shard_index, fragment.logical_offset, fragment.len));
} else if entry.members.is_empty()
&& !seen_logical_rel_paths.insert(entry.rel_path.clone())
{
return Err(RqError::Frame(format!(
"duplicate logical rel_path: {}",
entry.rel_path
)));
}
// E-15: a packed object carries a member offset table. Validate it
// off the wire (member paths, contiguity, and that the member lens
// tile the object exactly) so a hostile/malformed packed manifest
// fails closed before any decoder is allocated. The synthetic object
// `rel_path` (`.atp-pack-N`) is never committed; the member logical
// paths are what land on disk and must be unique + safe.
if !entry.members.is_empty() {
let mut expected_offset = 0u64;
for member in &entry.members {
validate_manifest_rel_path(&member.rel_path)?;
validate_manifest_sha256_hex(
"manifest packed member sha256_hex",
&member.sha256_hex,
)?;
if !seen_logical_rel_paths.insert(member.rel_path.clone()) {
return Err(RqError::Frame(format!(
"duplicate packed member rel_path: {}",
member.rel_path
)));
}
if member.offset != expected_offset {
return Err(RqError::Frame(format!(
"packed member {} offset {} is not contiguous (expected {expected_offset})",
member.rel_path, member.offset
)));
}
expected_offset = expected_offset.checked_add(member.len).ok_or_else(|| {
RqError::Frame(format!(
"packed member {} length overflow",
member.rel_path
))
})?;
}
if expected_offset != entry.size {
return Err(RqError::Frame(format!(
"packed members cover {expected_offset} bytes but object {} declares {}",
entry.rel_path, entry.size
)));
}
}
acc.checked_add(entry.size).ok_or_else(|| {
RqError::Frame("manifest declared size sum overflows u64".to_string())
})
})?;
if declared_total > config.max_transfer_bytes {
return Err(RqError::TooLarge {
size: declared_total,
max: config.max_transfer_bytes,
});
}
if single_file_fragmented && fragment_groups.len() != 1 {
return Err(RqError::Frame(format!(
"single-file fragmented manifest declares {} logical files",
fragment_groups.len()
)));
}
for (rel_path, group) in fragment_groups {
if !seen_logical_rel_paths.insert(rel_path.clone()) {
return Err(RqError::Frame(format!(
"duplicate logical rel_path: {rel_path}"
)));
}
if group.shards.len() != usize::try_from(group.shard_count).unwrap_or(usize::MAX) {
return Err(RqError::Frame(format!(
"fragment {rel_path} declares {} shards but manifest carries {}",
group.shard_count,
group.shards.len()
)));
}
let mut shards = group.shards;
shards.sort_by_key(|(shard_index, _, _)| *shard_index);
let mut expected_offset = 0u64;
for (position, (shard_index, offset, len)) in shards.iter().enumerate() {
if *shard_index != u32::try_from(position).unwrap_or(u32::MAX) {
return Err(RqError::Frame(format!(
"fragment {rel_path} has non-contiguous shard index {shard_index}"
)));
}
if *offset != expected_offset {
return Err(RqError::Frame(format!(
"fragment {rel_path} offset {offset} is not contiguous (expected {expected_offset})"
)));
}
expected_offset = expected_offset.checked_add(*len).ok_or_else(|| {
RqError::Frame(format!("fragment {rel_path} length sum overflows"))
})?;
}
if expected_offset != group.logical_size {
return Err(RqError::Frame(format!(
"fragment {rel_path} shards cover {expected_offset} bytes but logical size is {}",
group.logical_size
)));
}
}
Ok(())
}
fn validate_manifest_sha256_hex(label: &str, value: &str) -> Result<(), RqError> {
if value.len() != 64 || !value.bytes().all(|b| b.is_ascii_hexdigit()) {
return Err(RqError::Frame(format!("{label} must be 64 hex characters")));
}
Ok(())
}
fn sha256_hex_placeholder() -> String {
"0".repeat(64)
}
fn validate_manifest_rel_path(rel: &str) -> Result<(), RqError> {
if rel.is_empty() || rel.starts_with('/') || rel.starts_with('\\') {
return Err(RqError::Source(format!("unsafe manifest rel_path: {rel}")));
}
for component in rel.split('/') {
if component.is_empty()
|| component == "."
|| component == ".."
|| component.contains('\\')
|| component.contains(':')
{
return Err(RqError::Source(format!("unsafe manifest rel_path: {rel}")));
}
}
Ok(())
}
#[derive(Debug, Default)]
struct CompletionDigestIndex {
entry_digests: BTreeMap<u32, (u64, String)>,
logical_digests: BTreeMap<String, (u64, String)>,
merkle_root_hex: Option<String>,
}
impl CompletionDigestIndex {
fn from_round_complete(
complete: &RqRoundComplete,
manifest: &TransferManifest,
require_source_stream_trailer: bool,
) -> Result<Self, RqError> {
let mut index = Self::default();
if let Some(root) = &complete.merkle_root_hex {
validate_manifest_sha256_hex("ObjectComplete merkle_root_hex", root)?;
index.merkle_root_hex = Some(root.clone());
}
let manifest_indexes: BTreeSet<u32> =
manifest.entries.iter().map(|entry| entry.index).collect();
for digest in &complete.entry_digests {
validate_manifest_sha256_hex("ObjectComplete entry sha256_hex", &digest.sha256_hex)?;
if !manifest_indexes.contains(&digest.index) {
return Err(RqError::Frame(format!(
"ObjectComplete digest for unknown entry {}",
digest.index
)));
}
if index
.entry_digests
.insert(digest.index, (digest.size, digest.sha256_hex.clone()))
.is_some()
{
return Err(RqError::Frame(format!(
"duplicate ObjectComplete digest for entry {}",
digest.index
)));
}
}
for digest in &complete.logical_digests {
validate_manifest_rel_path(&digest.rel_path)?;
validate_manifest_sha256_hex("ObjectComplete logical sha256_hex", &digest.sha256_hex)?;
if index
.logical_digests
.insert(
digest.rel_path.clone(),
(digest.size, digest.sha256_hex.clone()),
)
.is_some()
{
return Err(RqError::Frame(format!(
"duplicate ObjectComplete logical digest for {}",
digest.rel_path
)));
}
}
if require_source_stream_trailer {
if index.merkle_root_hex.is_none() {
return Err(RqError::Frame(
"control source stream ObjectComplete missing merkle_root_hex".to_string(),
));
}
for entry in &manifest.entries {
if !index.entry_digests.contains_key(&entry.index) {
return Err(RqError::Frame(format!(
"control source stream ObjectComplete missing digest for entry {}",
entry.index
)));
}
}
}
Ok(index)
}
fn expected_entry<'a>(&'a self, entry: &'a ManifestEntry) -> (u64, &'a str) {
self.entry_digests
.get(&entry.index)
.map_or((entry.size, entry.sha256_hex.as_str()), |(size, sha)| {
(*size, sha.as_str())
})
}
fn expected_logical<'a>(&'a self, rel_path: &str, size: u64, sha: &'a str) -> (u64, &'a str) {
self.logical_digests
.get(rel_path)
.map_or((size, sha), |(trailer_size, trailer_sha)| {
(*trailer_size, trailer_sha.as_str())
})
}
fn expected_merkle_root<'a>(&'a self, manifest: &'a TransferManifest) -> &'a str {
self.merkle_root_hex
.as_deref()
.unwrap_or(&manifest.merkle_root_hex)
}
}
/// Join `base` with a forward-slash relative path, rejecting any component that
/// would escape `base`.
fn join_relative(base: &Path, rel: &str) -> Result<PathBuf, RqError> {
let mut out = base.to_path_buf();
for component in rel.split('/') {
if component.is_empty() || component == "." {
continue;
}
if component == ".." || component.contains('\\') || component.contains(':') {
return Err(RqError::Source(format!(
"unsafe path component in entry: {rel}"
)));
}
out.push(component);
}
Ok(out)
}
fn transfer_id_hex(merkle_root_hex: &str, total_bytes: u64, file_count: usize) -> String {
let mut hasher = Sha256::new();
hasher.update(b"asupersync.atp.rq.transfer-id.v1\0");
hasher.update(merkle_root_hex.as_bytes());
hasher.update(total_bytes.to_be_bytes());
hasher.update((file_count as u64).to_be_bytes());
hex_encode(&hasher.finalize()[..16])
}
fn transfer_id_hex_from_structure(
root_name: &str,
is_directory: bool,
total_bytes: u64,
entries: &[ManifestEntry],
) -> String {
let nonce = RQ_TRANSFER_SEQ.fetch_add(1, Ordering::Relaxed);
let mut hasher = Sha256::new();
hasher.update(b"asupersync.atp.rq.transfer-id.v3.structure\0");
hasher.update(root_name.as_bytes());
hasher.update([u8::from(is_directory)]);
hasher.update(total_bytes.to_be_bytes());
hasher.update((entries.len() as u64).to_be_bytes());
hasher.update(std::process::id().to_be_bytes());
hasher.update(nonce.to_be_bytes());
for entry in entries {
hasher.update(entry.index.to_be_bytes());
hasher.update(entry.rel_path.as_bytes());
hasher.update([0]);
hasher.update(entry.size.to_be_bytes());
if let Some(fragment) = &entry.fragment {
hasher.update(b"fragment\0");
hasher.update(fragment.rel_path.as_bytes());
hasher.update([0]);
hasher.update(fragment.shard_index.to_be_bytes());
hasher.update(fragment.shard_count.to_be_bytes());
hasher.update(fragment.logical_offset.to_be_bytes());
hasher.update(fragment.len.to_be_bytes());
hasher.update(fragment.logical_size.to_be_bytes());
}
for member in &entry.members {
hasher.update(b"member\0");
hasher.update(member.rel_path.as_bytes());
hasher.update([0]);
hasher.update(member.offset.to_be_bytes());
hasher.update(member.len.to_be_bytes());
}
}
hex_encode(&hasher.finalize()[..16])
}
// ─── UDP symbol datagram framing ─────────────────────────────────────────────
fn encode_symbol_datagram(
tag: u64,
entry: u32,
sym: &Symbol,
auth_tag: Option<&AuthenticationTag>,
) -> Vec<u8> {
let data = sym.data();
let auth_len = auth_tag.map_or(0, |_| TAG_SIZE);
let mut out = Vec::with_capacity(DGRAM_HEADER + auth_len + data.len());
out.extend_from_slice(&SYMBOL_MAGIC.to_be_bytes());
out.extend_from_slice(&tag.to_be_bytes());
out.extend_from_slice(&entry.to_be_bytes());
out.push(sym.id().sbn());
out.extend_from_slice(&sym.id().esi().to_be_bytes());
out.push(u8::from(sym.kind().is_repair()));
out.extend_from_slice(&u16::try_from(data.len()).unwrap_or(u16::MAX).to_be_bytes());
if let Some(auth_tag) = auth_tag {
out.extend_from_slice(auth_tag.as_bytes());
}
out.extend_from_slice(data);
out
}
#[derive(Debug, Clone, Copy)]
struct ParsedDatagram {
entry: u32,
sbn: u8,
esi: u32,
kind: SymbolKind,
auth_tag: Option<AuthenticationTag>,
payload_len: usize,
header_len: usize,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum SymbolDatagramParseError {
TruncatedHeader { len: usize, min: usize },
BadMagic { found: u32 },
WrongTransferTag { found: u64, expected: u64 },
PayloadTooLarge { declared: usize, max: usize },
TruncatedPayload { len: usize, min: usize },
}
fn parse_symbol_header_checked(
buf: &[u8],
expect_tag: u64,
auth_required: bool,
max_payload_len: Option<usize>,
) -> Result<ParsedDatagram, SymbolDatagramParseError> {
let header_len = if auth_required {
AUTH_DGRAM_HEADER
} else {
DGRAM_HEADER
};
if buf.len() < header_len {
return Err(SymbolDatagramParseError::TruncatedHeader {
len: buf.len(),
min: header_len,
});
}
let found_magic = u32::from_be_bytes([buf[0], buf[1], buf[2], buf[3]]);
if found_magic != SYMBOL_MAGIC {
return Err(SymbolDatagramParseError::BadMagic { found: found_magic });
}
let tag = u64::from_be_bytes([
buf[4], buf[5], buf[6], buf[7], buf[8], buf[9], buf[10], buf[11],
]);
if tag != expect_tag {
return Err(SymbolDatagramParseError::WrongTransferTag {
found: tag,
expected: expect_tag,
});
}
let entry = u32::from_be_bytes([buf[12], buf[13], buf[14], buf[15]]);
let sbn = buf[16];
let esi = u32::from_be_bytes([buf[17], buf[18], buf[19], buf[20]]);
let kind = if buf[21] == 0 {
SymbolKind::Source
} else {
SymbolKind::Repair
};
let payload_len = usize::from(u16::from_be_bytes([buf[22], buf[23]]));
if let Some(max) = max_payload_len
&& payload_len > max
{
return Err(SymbolDatagramParseError::PayloadTooLarge {
declared: payload_len,
max,
});
}
let auth_tag = if auth_required {
let mut tag_bytes = [0u8; TAG_SIZE];
tag_bytes.copy_from_slice(&buf[DGRAM_HEADER..AUTH_DGRAM_HEADER]);
Some(AuthenticationTag::from_bytes(tag_bytes))
} else {
None
};
let min = header_len + payload_len;
if buf.len() < min {
return Err(SymbolDatagramParseError::TruncatedPayload {
len: buf.len(),
min,
});
}
Ok(ParsedDatagram {
entry,
sbn,
esi,
kind,
auth_tag,
payload_len,
header_len,
})
}
fn parse_symbol_header(buf: &[u8], expect_tag: u64, auth_required: bool) -> Option<ParsedDatagram> {
parse_symbol_header_checked(buf, expect_tag, auth_required, None).ok()
}
/// Fuzz-visible symbol-datagram parser result.
#[cfg(any(test, feature = "fuzz"))]
#[doc(hidden)]
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct RqSymbolDatagramFuzzParse {
/// Manifest entry index carried by the datagram.
pub entry: u32,
/// RaptorQ source-block number.
pub sbn: u8,
/// RaptorQ encoding-symbol id.
pub esi: u32,
/// Whether the datagram carries a repair symbol.
pub is_repair: bool,
/// Optional per-symbol authentication tag bytes.
pub auth_tag: Option<[u8; TAG_SIZE]>,
/// Offset where the symbol payload begins.
pub payload_offset: usize,
/// Declared symbol payload length.
pub payload_len: usize,
}
/// Typed fuzz-visible parser error for ATP-over-RaptorQ UDP symbol datagrams.
#[cfg(any(test, feature = "fuzz"))]
#[doc(hidden)]
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum RqSymbolDatagramFuzzError {
/// The datagram ended before the required header.
TruncatedHeader {
/// Observed byte length.
len: usize,
/// Minimum required byte length.
min: usize,
},
/// The magic prefix was not `ATRQ`.
BadMagic {
/// Observed magic value.
found: u32,
},
/// The transfer tag did not match the expected transfer.
WrongTransferTag {
/// Observed transfer tag.
found: u64,
/// Expected transfer tag.
expected: u64,
},
/// The declared payload length exceeds the fuzz harness budget.
PayloadTooLarge {
/// Declared payload length.
declared: usize,
/// Maximum payload length accepted by the harness.
max: usize,
},
/// The datagram ended before the declared payload bytes.
TruncatedPayload {
/// Observed byte length.
len: usize,
/// Minimum required byte length.
min: usize,
},
}
#[cfg(any(test, feature = "fuzz"))]
impl From<SymbolDatagramParseError> for RqSymbolDatagramFuzzError {
fn from(error: SymbolDatagramParseError) -> Self {
match error {
SymbolDatagramParseError::TruncatedHeader { len, min } => {
Self::TruncatedHeader { len, min }
}
SymbolDatagramParseError::BadMagic { found } => Self::BadMagic { found },
SymbolDatagramParseError::WrongTransferTag { found, expected } => {
Self::WrongTransferTag { found, expected }
}
SymbolDatagramParseError::PayloadTooLarge { declared, max } => {
Self::PayloadTooLarge { declared, max }
}
SymbolDatagramParseError::TruncatedPayload { len, min } => {
Self::TruncatedPayload { len, min }
}
}
}
}
/// Parse an ATP-over-RaptorQ UDP symbol datagram with typed errors for fuzzing.
#[cfg(any(test, feature = "fuzz"))]
#[doc(hidden)]
pub fn parse_symbol_datagram_for_fuzz(
buf: &[u8],
expect_tag: u64,
auth_required: bool,
max_payload_len: usize,
) -> Result<RqSymbolDatagramFuzzParse, RqSymbolDatagramFuzzError> {
let parsed =
parse_symbol_header_checked(buf, expect_tag, auth_required, Some(max_payload_len))?;
Ok(RqSymbolDatagramFuzzParse {
entry: parsed.entry,
sbn: parsed.sbn,
esi: parsed.esi,
is_repair: parsed.kind.is_repair(),
auth_tag: parsed.auth_tag.map(|tag| *tag.as_bytes()),
payload_offset: parsed.header_len,
payload_len: parsed.payload_len,
})
}
// ─── Per-entry coding state ──────────────────────────────────────────────────
/// Compute the source-symbol count for an entry of `size` bytes given the
/// symbol size (`ceil(size / symbol_size)`, with a 1-symbol floor for empties).
#[cfg(test)]
fn source_symbol_count(size: u64, symbol_size: u16) -> usize {
let s = u64::from(symbol_size.max(1));
usize::try_from(size.div_ceil(s).max(1)).unwrap_or(usize::MAX)
}
#[cfg(test)]
fn max_block_source_symbol_count(size: u64, symbol_size: u16, max_block_size: usize) -> usize {
if size == 0 {
return 1;
}
let s = usize::from(symbol_size.max(1));
let capped_block = usize::try_from(size)
.unwrap_or(usize::MAX)
.min(max_block_size.max(1));
capped_block.div_ceil(s).max(1)
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
struct EncodeAheadBlock {
sbn: u8,
start: usize,
len: usize,
k: usize,
}
#[derive(Debug)]
struct EncodeAheadSymbol {
entry: u32,
symbol: Symbol,
}
impl EncodeAheadSymbol {
fn from_encoded(entry: u32, encoded: EncodedSymbol) -> Self {
Self {
entry,
symbol: encoded.into_symbol(),
}
}
}
#[derive(Debug, Default)]
struct EncodeAheadRing {
slot: Option<EncodeAheadSymbol>,
}
impl EncodeAheadRing {
const CAPACITY: usize = 1;
fn push(&mut self, symbol: EncodeAheadSymbol) -> Result<(), RqError> {
if self.slot.is_some() {
return Err(RqError::Coding(format!(
"M={} encode-ahead ring is full",
Self::CAPACITY
)));
}
self.slot = Some(symbol);
Ok(())
}
fn pop(&mut self) -> Option<EncodeAheadSymbol> {
self.slot.take()
}
fn is_empty(&self) -> bool {
self.slot.is_none()
}
}
fn encode_ahead_blocks(
bytes_len: usize,
config: &RqConfig,
) -> Result<Vec<EncodeAheadBlock>, RqError> {
let symbol_size = usize::from(config.symbol_size);
if symbol_size == 0 {
return Err(RqError::Coding(
"invalid configuration: symbol_size must be non-zero".to_string(),
));
}
let max_block_size = config.max_block_size;
if max_block_size == 0 {
return Err(RqError::Coding(
"invalid configuration: max_block_size must be non-zero".to_string(),
));
}
if bytes_len == 0 {
return Ok(Vec::new());
}
let max_total = max_object_size(max_block_size);
if bytes_len > max_total {
return Err(RqError::TooLarge {
size: u64::try_from(bytes_len).unwrap_or(u64::MAX),
max: u64::try_from(max_total).unwrap_or(u64::MAX),
});
}
let mut blocks = Vec::new();
let mut start = 0usize;
while start < bytes_len {
if blocks.len() >= MAX_SOURCE_BLOCKS {
return Err(RqError::TooLarge {
size: u64::try_from(bytes_len).unwrap_or(u64::MAX),
max: u64::try_from(max_total).unwrap_or(u64::MAX),
});
}
let sbn = u8::try_from(blocks.len()).map_err(|_| {
RqError::Coding("encode-ahead source block number overflow".to_string())
})?;
let len = (bytes_len - start).min(max_block_size);
let k = len.div_ceil(symbol_size);
blocks.push(EncodeAheadBlock { sbn, start, len, k });
start += len;
}
Ok(blocks)
}
fn effective_transfer_max_block_size(
config: &RqConfig,
entries: &[EntryDigest],
) -> Result<usize, RqError> {
let mut max_entry_len = 0usize;
for entry in entries {
let len = usize::try_from(entry.size).map_err(|_| RqError::TooLarge {
size: entry.size,
max: u64::try_from(usize::MAX).unwrap_or(u64::MAX),
})?;
max_entry_len = max_entry_len.max(len);
}
effective_max_block_size_for_largest_entry(config, max_entry_len)
}
pub(in crate::net::atp) fn effective_max_block_size_for_largest_entry(
config: &RqConfig,
max_entry_len: usize,
) -> Result<usize, RqError> {
let symbol_size = usize::from(config.symbol_size.max(1));
let configured_max = config.max_block_size.max(symbol_size);
// E-12: large logical files must be split into bounded RaptorQ objects before
// this transfer-wide block size is chosen. If an unsplit entry still exceeds
// the one-byte SBN envelope, fail closed instead of raising K and making lossy
// huge-file decode quadratic.
let max_supported = max_object_size(configured_max);
if max_entry_len > max_supported {
return Err(RqError::Coding(format!(
"[ASUP-E803] ATP block-size planning failed: largest entry {max_entry_len} bytes exceeds supported max {max_supported} bytes"
)));
}
let target = symbol_size
.saturating_mul(TARGET_SOURCE_SYMBOLS_PER_BLOCK)
.min(TARGET_STREAMING_BLOCK_BYTES);
let min_for_block_limit = max_entry_len
.div_ceil(MAX_SOURCE_BLOCKS)
.max(symbol_size)
.div_ceil(symbol_size)
.saturating_mul(symbol_size);
// For entries within `256 * configured_max` (<= ~2 GiB at defaults),
// `min_for_block_limit <= configured_max`, so this preserves the bounded
// streaming target while honoring the SBN envelope.
Ok(target
.max(min_for_block_limit)
.min(configured_max)
.max(symbol_size))
}
/// Sender-side encoder state for one entry. Holds only source metadata; each
/// encode-ahead block is read on demand so the sender never retains the whole
/// object in memory.
struct EntryEncoder {
index: u32,
object_id: ObjectId,
abs_path: PathBuf,
source_offset: usize,
size: usize,
/// Cumulative repair symbols already requested from the encoder, indexed by
/// source block. Feedback rounds request more and send only the newly-minted
/// ones at their TRUE encoder ESIs — a RaptorQ repair symbol's payload is
/// bound to its ESI, so it must never be relabeled.
repair_cursors: Vec<usize>,
}
/// Receiver-side decoder state for one entry.
struct EntryDecoder {
index: u32,
object_id: ObjectId,
size: u64,
/// `Option` so completed entries can drop decoder state after streaming all
/// blocks to disk.
pipeline: Option<DecodingPipeline>,
complete: bool,
staging_path: PathBuf,
/// Entry-relative writes are offset by this amount inside `staging_path`.
/// Normal entries keep zero; single-file large fragments share one logical
/// staging file and write at their logical offset.
staging_write_offset: u64,
/// Size to pre-create for `staging_path`. This is usually `size`; shared
/// fragment staging uses the whole logical file size.
staging_file_len: u64,
/// Allows multiple fragment decoders to open the same pre-created logical
/// staging file instead of treating `AlreadyExists` as hostile.
staging_shared: bool,
staging_created: bool,
staging_file: Option<crate::fs::File>,
staging_cursor: Option<u64>,
staging_unflushed_bytes: usize,
cache_staging_file: bool,
bytes_written: u64,
max_block_size: usize,
source_streaming: bool,
source_blocks: Vec<SourceBlockProgress>,
pending_decodes: Vec<PendingDecode>,
source_write_buffer: Vec<u8>,
source_write_buffer_offset: Option<u64>,
/// Incremental SHA-256 + content-id over the control-source-stream bytes,
/// folded in receive order (the source-stream path enforces strictly
/// in-order contiguous writes, so this equals the post-stream digest).
/// `Some` only on the reliable control-source-stream path; the lossy
/// RaptorQ-datagram path leaves this `None` and verifies post-stream.
inc: Option<crate::net::atp::transport_common::StagedEntryReceive>,
/// Finalized `(size, content_id, content_sha256)` once the entry completes,
/// letting commit skip the post-stream staging-file re-read+hash.
inc_digest: Option<(u64, crate::atp::object::ObjectId, [u8; 32])>,
}
struct PendingDecode {
block_sbn: u8,
handle: crate::runtime::TaskHandle<BlockDecodeOutcome>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum DecodeDispatch {
Queued,
NoProgress,
}
#[derive(Debug, Clone, Copy)]
struct DecodeDispatchOutcome {
dispatch: DecodeDispatch,
decode_stats: RqDecodeRoundStats,
}
impl DecodeDispatchOutcome {
fn new(dispatch: DecodeDispatch, decode_stats: RqDecodeRoundStats) -> Self {
Self {
dispatch,
decode_stats,
}
}
}
#[derive(Debug, Default, Clone, Copy)]
struct RqSymbolFeed {
accepted: bool,
source_auth_micros: u64,
source_persist_micros: u64,
pipeline_feed_micros: u64,
block_persist_micros: u64,
decode_dispatch_micros: u64,
source_seed_micros: u64,
decode_stats: RqDecodeRoundStats,
}
#[derive(Debug, Default, Clone, Copy)]
struct SourceStreamingSeedStats {
seeded: u64,
decode_stats: RqDecodeRoundStats,
}
fn should_cache_entry_staging_file(
entry_size: u64,
manifest_entries: usize,
packed_members: usize,
) -> bool {
let bounded_manifest = manifest_entries <= ENTRY_STAGING_FILE_CACHE_MAX_ENTRIES;
let large_entry_cache = entry_size >= ENTRY_STAGING_FILE_CACHE_MIN_BYTES && bounded_manifest;
let packed_tree_batch = packed_members > 0 && entry_size > 0 && bounded_manifest;
large_entry_cache || packed_tree_batch
}
fn is_round_scoped_entry_staging_cache(dec: &EntryDecoder) -> bool {
dec.cache_staging_file && dec.size < ENTRY_STAGING_FILE_CACHE_MIN_BYTES
}
fn entry_source_block_count_for_geometry(
entry_size: u64,
max_block_size: usize,
observed_source_blocks: usize,
) -> usize {
let max_block_size = u64::try_from(max_block_size.max(1)).unwrap_or(u64::MAX);
let planned_blocks = usize::try_from(entry_size.div_ceil(max_block_size)).unwrap_or(usize::MAX);
observed_source_blocks.max(planned_blocks).max(1)
}
fn entry_source_block_count(dec: &EntryDecoder) -> usize {
entry_source_block_count_for_geometry(dec.size, dec.max_block_size, dec.source_blocks.len())
}
fn source_streaming_entry_complete(dec: &EntryDecoder) -> bool {
!dec.source_blocks.is_empty() && dec.bytes_written == dec.size
}
fn single_file_fragment_staging_path(
manifest: &TransferManifest,
staging_dir: &Path,
) -> Option<PathBuf> {
if manifest.is_directory || manifest.entries.is_empty() {
return None;
}
let first_rel_path = manifest
.entries
.first()?
.fragment
.as_ref()?
.rel_path
.as_str();
let all_same_fragment = manifest.entries.iter().all(|entry| {
entry
.fragment
.as_ref()
.is_some_and(|fragment| fragment.rel_path == first_rel_path)
});
all_same_fragment.then(|| {
staging_dir
.join(RQ_SINGLE_FILE_FRAGMENT_STAGING_DIR)
.join("0")
})
}
fn receive_staging_layout_for_entry(
entry: &ManifestEntry,
staging_dir: &Path,
single_file_fragment_staging_path: Option<&Path>,
) -> (PathBuf, u64, u64, bool) {
if let (Some(fragment), Some(staging_path)) =
(entry.fragment.as_ref(), single_file_fragment_staging_path)
{
return (
staging_path.to_path_buf(),
fragment.logical_offset,
fragment.logical_size,
true,
);
}
(
staging_dir.join(entry.index.to_string()),
0,
entry.size,
false,
)
}
fn entry_staging_absolute_offset(
dec: &EntryDecoder,
offset: u64,
len: usize,
) -> Result<u64, RqError> {
let len = u64::try_from(len).unwrap_or(u64::MAX);
let entry_end = offset
.checked_add(len)
.ok_or_else(|| RqError::Coding(format!("entry {} staging range overflow", dec.index)))?;
if entry_end > dec.size {
return Err(RqError::Frame(format!(
"entry {} staging write range {}..{} overruns declared size {}",
dec.index, offset, entry_end, dec.size
)));
}
let absolute = dec
.staging_write_offset
.checked_add(offset)
.ok_or_else(|| {
RqError::Coding(format!(
"entry {} shared staging offset overflow",
dec.index
))
})?;
let absolute_end = absolute.checked_add(len).ok_or_else(|| {
RqError::Coding(format!("entry {} shared staging range overflow", dec.index))
})?;
if absolute_end > dec.staging_file_len {
return Err(RqError::Frame(format!(
"entry {} staging write range {}..{} overruns staging file size {}",
dec.index, absolute, absolute_end, dec.staging_file_len
)));
}
Ok(absolute)
}
fn decoder_position_for_entry(decoders: &[EntryDecoder], entry: u32) -> Option<usize> {
let direct = usize::try_from(entry).ok()?;
if decoders
.get(direct)
.is_some_and(|decoder| decoder.index == entry)
{
return Some(direct);
}
decoders.iter().position(|decoder| decoder.index == entry)
}
#[cfg(test)]
fn should_parallel_decode_entry_geometry(
entry_size: u64,
max_block_size: usize,
observed_source_blocks: usize,
) -> bool {
entry_size >= RQ_PARALLEL_DECODE_MIN_ENTRY_BYTES
&& entry_source_block_count_for_geometry(entry_size, max_block_size, observed_source_blocks)
>= RQ_PARALLEL_DECODE_MIN_SOURCE_BLOCKS
}
fn should_parallel_decode_entry(dec: &EntryDecoder) -> bool {
dec.size >= RQ_PARALLEL_DECODE_MIN_ENTRY_BYTES
&& entry_source_block_count(dec) >= RQ_PARALLEL_DECODE_MIN_SOURCE_BLOCKS
}
fn transfer_decode_size_gate(decoders: &[EntryDecoder]) -> usize {
if decoders.is_empty() || decoders.iter().any(should_parallel_decode_entry) {
RQ_MAX_PENDING_DECODE_JOBS_PER_TRANSFER_HARD
} else {
0
}
}
fn entry_decode_width_budget(dec: &EntryDecoder, transfer_decode_width: usize) -> usize {
if !should_parallel_decode_entry(dec) {
return 0;
}
let block_count = entry_source_block_count(dec);
block_count
.min(RQ_MAX_PENDING_DECODE_JOBS_PER_ENTRY)
.min(transfer_decode_width.max(1))
.max(1)
}
#[cfg(test)]
fn entry_decode_width_budget_for_geometry(
entry_size: u64,
max_block_size: usize,
observed_source_blocks: usize,
transfer_decode_width: usize,
) -> usize {
if !should_parallel_decode_entry_geometry(entry_size, max_block_size, observed_source_blocks) {
return 0;
}
let block_count =
entry_source_block_count_for_geometry(entry_size, max_block_size, observed_source_blocks);
block_count
.min(RQ_MAX_PENDING_DECODE_JOBS_PER_ENTRY)
.min(transfer_decode_width.max(1))
.max(1)
}
fn can_spawn_parallel_decode(pending_decodes: usize, entry_decode_width: usize) -> bool {
entry_decode_width > 1 && pending_decodes < entry_decode_width
}
fn rq_decode_job_memory_estimate_bytes(max_block_size: usize, symbol_size: u16) -> usize {
let retained_symbol_bytes = rq_max_buffered_symbols_per_block(max_block_size, symbol_size)
.saturating_mul(usize::from(symbol_size.max(1)));
retained_symbol_bytes
.saturating_mul(RQ_DECODE_JOB_SYMBOL_MEMORY_MULTIPLIER)
.max(RQ_DECODE_JOB_MEMORY_FLOOR_BYTES)
}
#[derive(Debug, Clone, Copy)]
struct RqDecodeWidthBudget {
effective: usize,
core_limit: usize,
memory_limit: usize,
job_memory_bytes: usize,
max_block_size: usize,
}
fn rq_decode_reserved_io_cores(available: usize) -> usize {
if available <= 1 {
return 0;
}
(available / 4)
.clamp(
RQ_DECODE_MIN_CORES_RESERVED_FOR_IO,
RQ_DECODE_MAX_CORES_RESERVED_FOR_IO,
)
.min(available.saturating_sub(1))
}
fn rq_decode_core_limit_for_available(available: usize) -> usize {
available
.saturating_sub(rq_decode_reserved_io_cores(available))
.max(1)
.min(RQ_MAX_PENDING_DECODE_JOBS_PER_TRANSFER_HARD)
}
fn rq_decode_core_limit() -> usize {
static CORE_LIMIT: std::sync::OnceLock<usize> = std::sync::OnceLock::new();
*CORE_LIMIT.get_or_init(|| {
let available = std::thread::available_parallelism().map_or(1, std::num::NonZeroUsize::get);
rq_decode_core_limit_for_available(available)
})
}
fn rq_decode_core_limit_for_cx(cx: &Cx) -> usize {
// ATP CLI/daemon runtimes size a blocking pool explicitly for CPU-heavy
// RaptorQ work. Prefer that live cap over process CPU discovery so cgroup
// or container affinity quirks do not collapse receiver decode to one lane.
cx.blocking_pool_handle()
.map_or_else(rq_decode_core_limit, |pool| {
rq_decode_core_limit_for_available(pool.current_max_threads())
})
}
fn rq_decode_width_budget_snapshot_for_core_limit(
decoders: &[EntryDecoder],
symbol_size: u16,
core_limit: usize,
) -> RqDecodeWidthBudget {
let max_block_size = decoders
.iter()
.map(|decoder| decoder.max_block_size)
.max()
.unwrap_or(DEFAULT_MAX_BLOCK_SIZE);
let job_memory_bytes = rq_decode_job_memory_estimate_bytes(max_block_size, symbol_size);
let memory_limit = RQ_DECODE_JOB_MEMORY_BUDGET_BYTES
.checked_div(job_memory_bytes)
.unwrap_or(1)
.max(1);
let size_gate = transfer_decode_size_gate(decoders);
RqDecodeWidthBudget {
effective: core_limit.min(memory_limit).min(size_gate),
core_limit,
memory_limit,
job_memory_bytes,
max_block_size,
}
}
#[cfg(test)]
fn rq_decode_width_budget_snapshot(
decoders: &[EntryDecoder],
symbol_size: u16,
) -> RqDecodeWidthBudget {
rq_decode_width_budget_snapshot_for_core_limit(decoders, symbol_size, rq_decode_core_limit())
}
fn rq_decode_width_budget_snapshot_for_cx(
cx: &Cx,
decoders: &[EntryDecoder],
symbol_size: u16,
) -> RqDecodeWidthBudget {
rq_decode_width_budget_snapshot_for_core_limit(
decoders,
symbol_size,
rq_decode_core_limit_for_cx(cx),
)
}
fn rq_decode_width_budget_for_cx(cx: &Cx, decoders: &[EntryDecoder], symbol_size: u16) -> usize {
rq_decode_width_budget_snapshot_for_cx(cx, decoders, symbol_size).effective
}
fn block_decode_pending(dec: &EntryDecoder, block_sbn: u8) -> bool {
dec.pending_decodes
.iter()
.any(|pending| pending.block_sbn == block_sbn)
}
fn rq_pending_decode_jobs(decoders: &[EntryDecoder]) -> usize {
decoders
.iter()
.map(|decoder| decoder.pending_decodes.len())
.sum()
}
fn source_streaming_block_ready_to_seed(dec: &EntryDecoder, sbn: usize) -> bool {
let Some(block) = dec.source_blocks.get(sbn) else {
return false;
};
if block.complete {
return false;
}
let Ok(block_sbn) = u8::try_from(sbn) else {
return false;
};
let Some(status) = dec
.pipeline
.as_ref()
.and_then(|pipeline| pipeline.block_status(block_sbn))
else {
return false;
};
let unseeded_sources = block
.received
.iter()
.zip(&block.pipeline_seeded)
.filter(|(received, seeded)| **received && !**seeded)
.count();
status.symbols_received.saturating_add(unseeded_sources) >= block.k
}
fn source_seed_symbol_plan(
dec: &EntryDecoder,
sbn: usize,
esi: usize,
symbol_size: usize,
) -> Result<Option<(usize, usize, Option<AuthenticationTag>)>, RqError> {
let Some(block) = dec.source_blocks.get(sbn) else {
return Ok(None);
};
if esi >= block.k || !block.received[esi] || block.pipeline_seeded[esi] || block.complete {
return Ok(None);
}
let Some(within_block) = esi.checked_mul(symbol_size) else {
return Err(RqError::Coding(format!(
"entry {} source seed offset overflow",
dec.index
)));
};
if within_block >= block.len {
return Ok(None);
}
let take = symbol_size.min(block.len - within_block);
Ok(Some((within_block, take, block.auth_tags[esi])))
}
fn rq_max_buffered_symbols_per_block(max_block_size: usize, symbol_size: u16) -> usize {
let symbol_size = usize::from(symbol_size.max(1));
let k = max_block_size.div_ceil(symbol_size).max(1);
let repair_extra = k.max(RQ_REPAIR_SYMBOL_RETENTION_MIN_EXTRA);
k.saturating_add(repair_extra)
}
#[derive(Debug)]
struct SourceBlockProgress {
start: u64,
len: usize,
k: usize,
received: Vec<bool>,
pipeline_seeded: Vec<bool>,
auth_tags: Vec<Option<AuthenticationTag>>,
received_count: usize,
complete: bool,
}
/// Best-effort backstop for receive staging directories.
///
/// The RQ receiver creates a per-transfer staging directory before it starts
/// accepting untrusted UDP symbols. Normal and error exits should not leave
/// hidden payload fragments under the destination, and cancellation can drop the
/// future before it reaches a cooperative return path. This mirrors the TCP
/// transport's staging guard.
struct RqStagingDirGuard {
dir: PathBuf,
}
impl RqStagingDirGuard {
fn new(dir: PathBuf) -> Self {
Self { dir }
}
}
impl Drop for RqStagingDirGuard {
fn drop(&mut self) {
let _ = std::fs::remove_dir_all(&self.dir);
}
}
// ─── Public API: send ────────────────────────────────────────────────────────
/// Transfer the file or directory at `source` to `addr` (the receiver's TCP
/// control address) using RaptorQ symbols over UDP.
///
/// Returns the receiver's verified receipt. Fails closed on an unreachable peer,
/// a rejected handshake, a size-limit breach, a fountain loop that does not
/// converge, or a receiver integrity rejection.
pub async fn send_path(
cx: &Cx,
addr: SocketAddr,
source: &Path,
mut config: RqConfig,
peer_id: &str,
) -> Result<SendReport, RqError> {
cx.checkpoint().map_err(|_| RqError::Cancelled)?;
let symbol_auth = config.symbol_auth_context()?;
let symbol_auth_enabled = symbol_auth.is_some();
let (root_name, is_directory, raw_entries) = collect_entries(source).await?;
let preflight_total_bytes = source_entries_total_bytes(&raw_entries).await?;
if preflight_total_bytes > config.max_transfer_bytes {
return Err(RqError::TooLarge {
size: preflight_total_bytes,
max: config.max_transfer_bytes,
});
}
let prefer_control_source_stream =
control_source_stream_eligible(preflight_total_bytes, &config);
if prefer_control_source_stream {
let stream =
match crate::time::timeout(cx.now(), DEFAULT_CONNECT_TIMEOUT, TcpStream::connect(addr))
.await
{
Ok(Ok(stream)) => stream,
Ok(Err(err)) => return Err(RqError::Io(err)),
Err(_elapsed) => {
return Err(RqError::Io(std::io::Error::new(
std::io::ErrorKind::TimedOut,
format!("connect to {addr} timed out after {DEFAULT_CONNECT_TIMEOUT:?}"),
)));
}
};
tune_control_stream_for_bulk_source(&stream);
let peer = stream.peer_addr().unwrap_or(addr);
let mut control = FrameTransport::new(stream);
control
.send(&json_frame(
FrameType::Handshake,
&Hello {
protocol: ATP_RQ_PROTOCOL,
role: "sender".to_string(),
peer_id: peer_id.to_string(),
symbol_size: config.symbol_size,
max_block_size: config.max_block_size as u64,
symbol_auth: symbol_auth_enabled,
total_bytes: preflight_total_bytes,
prefer_control_source_stream,
},
)?)
.await?;
let ack_frame = control.recv().await?;
if ack_frame.frame_type() != FrameType::HandshakeAck {
return Err(RqError::Unexpected {
got: ack_frame.frame_type(),
expected: "HandshakeAck",
});
}
let ack: HelloAck = parse_json(&ack_frame)?;
if !ack.accepted {
return Err(RqError::HandshakeRejected(
ack.reason.unwrap_or_else(|| "no reason given".to_string()),
));
}
if !ack.control_source_stream {
return Err(RqError::HandshakeRejected(
"receiver declined required control source stream".to_string(),
));
}
let prepared = prepare_control_source_transfer(
root_name,
is_directory,
raw_entries,
&config,
preflight_total_bytes,
)
.await?;
let transfer_id = prepared.manifest.transfer_id.clone();
let total_bytes = prepared.manifest.total_bytes;
let mut encoders: Vec<EntryEncoder> = Vec::with_capacity(prepared.entries.len());
for (i, (entry, size)) in prepared
.entries
.iter()
.zip(prepared.object_sizes.iter())
.enumerate()
{
let index = u32::try_from(i).unwrap_or(u32::MAX);
let size = usize::try_from(*size).map_err(|_| RqError::TooLarge {
size: *size,
max: u64::try_from(usize::MAX).unwrap_or(u64::MAX),
})?;
let source_offset =
usize::try_from(entry.source_offset).map_err(|_| RqError::TooLarge {
size: entry.source_offset,
max: u64::try_from(usize::MAX).unwrap_or(u64::MAX),
})?;
encoders.push(EntryEncoder {
index,
object_id: entry_object_id(&transfer_id, index),
abs_path: entry.abs_path.clone(),
source_offset,
size,
repair_cursors: Vec::new(),
});
}
control
.send(&json_frame(FrameType::ObjectManifest, &prepared.manifest)?)
.await?;
let digest_report = stream_control_source_entries(
cx,
&mut control,
&encoders,
&prepared.manifest,
&prepared.precomputed_logical_digests,
&transfer_id,
symbol_auth.as_ref(),
)
.await?;
if digest_report.bytes_streamed != total_bytes {
return Err(RqError::Source(format!(
"control source stream sent {} bytes, expected {total_bytes}",
digest_report.bytes_streamed
)));
}
let files = u32::try_from(digest_report.logical_digests.len()).unwrap_or(u32::MAX);
let merkle_root_hex = digest_report.merkle_root_hex.clone();
control
.send(&json_frame(
FrameType::ObjectComplete,
&RqRoundComplete {
round_symbols_sent: 0,
entry_digests: digest_report.entry_digests,
logical_digests: digest_report.logical_digests,
merkle_root_hex: Some(merkle_root_hex.clone()),
},
)?)
.await?;
let reply = control.recv().await?;
if reply.frame_type() != FrameType::Proof {
return Err(RqError::Unexpected {
got: reply.frame_type(),
expected: "Proof",
});
}
let receipt: ReceiveReceipt = parse_json(&reply)?;
let _ = control
.send(&Frame::empty(FrameType::Close).map_err(|e| RqError::Frame(e.to_string()))?)
.await;
if !receipt.committed {
return Err(RqError::Integrity(
receipt
.reason
.clone()
.unwrap_or_else(|| "receiver did not commit".to_string()),
));
}
return Ok(SendReport {
transfer_id,
bytes_sent: total_bytes,
files,
symbols_sent: 0,
feedback_rounds: 0,
merkle_root_hex,
receipt,
udp_send_acceleration: UdpSendAccelerationReport::default(),
peer,
});
}
// E-15: coalesce sub-threshold files into fewer/larger combined RaptorQ
// objects. `entries` are the OBJECTS to spray (a packed entry's `abs_path`
// points at a temp file holding the member concatenation); `logical_digests`
// are the per-LOGICAL-FILE digests that drive the merkle root. For the
// no-packing case `entries == raw_entries` and `logical_digests` equals the
// per-file digests, so everything is byte-identical to a prior transfer. The
// temp dir owns every pack temp file and must outlive the spray loop below.
let (packed_entries, logical_digests, _pack_tempdir) =
pack_small_files(raw_entries, &config).await?;
let entries = split_large_entries(packed_entries, &logical_digests, &config).await?;
let mut hash_buf = vec![0u8; RQ_STREAM_HASH_BUFFER_SIZE];
// Per-OBJECT digests: size + sha of each entry's `abs_path` (the temp file for
// packed entries). These feed the manifest entry size/sha (object-level verify)
// and the effective block size — they describe the RaptorQ objects on the wire.
let mut digests = Vec::with_capacity(entries.len());
let mut total_bytes = 0u64;
for entry in &entries {
let (size, content_id, content_sha256) = hash_source_entry_streaming(entry, &mut hash_buf)
.await
.map_err(|e| RqError::Source(e.into_message()))?;
total_bytes = total_bytes.checked_add(size).ok_or(RqError::TooLarge {
size: u64::MAX,
max: config.max_transfer_bytes,
})?;
if total_bytes > config.max_transfer_bytes {
return Err(RqError::TooLarge {
size: total_bytes,
max: config.max_transfer_bytes,
});
}
digests.push(EntryDigest {
rel_path: entry.rel_path.clone(),
size,
content_id,
content_sha256,
});
}
config.max_block_size = effective_transfer_max_block_size(&config, &digests)?;
// Merkle root is over the LOGICAL files (members flattened), identical on both
// sides regardless of how files were packed into objects.
let merkle_root_hex = flat_merkle_root_from_digests(&logical_digests);
let manifest_entries: Vec<ManifestEntry> = entries
.iter()
.zip(digests.iter())
.enumerate()
.map(|(i, (entry, digest))| ManifestEntry {
index: u32::try_from(i).unwrap_or(u32::MAX),
rel_path: digest.rel_path.clone(),
size: digest.size,
sha256_hex: hex_encode(&digest.content_sha256),
members: entry.members.clone(),
fragment: entry.fragment.clone(),
})
.collect();
let packed_objects = manifest_entries
.iter()
.filter(|e| !e.members.is_empty())
.count();
rqtrace!(
"sender: E-15 pack: {} logical files -> {} RaptorQ objects ({} packed)",
logical_digests.len(),
manifest_entries.len(),
packed_objects
);
let transfer_id = transfer_id_hex(&merkle_root_hex, total_bytes, manifest_entries.len());
let tag = transfer_tag(&transfer_id);
let manifest = TransferManifest {
transfer_id: transfer_id.clone(),
root_name,
is_directory,
total_bytes,
merkle_root_hex: merkle_root_hex.clone(),
entries: manifest_entries,
};
let prefer_control_source_stream = control_source_stream_eligible(total_bytes, &config);
// Control plane: TCP connect + handshake.
let stream =
match crate::time::timeout(cx.now(), DEFAULT_CONNECT_TIMEOUT, TcpStream::connect(addr))
.await
{
Ok(Ok(stream)) => stream,
Ok(Err(err)) => return Err(RqError::Io(err)),
Err(_elapsed) => {
return Err(RqError::Io(std::io::Error::new(
std::io::ErrorKind::TimedOut,
format!("connect to {addr} timed out after {DEFAULT_CONNECT_TIMEOUT:?}"),
)));
}
};
if prefer_control_source_stream {
tune_control_stream_for_bulk_source(&stream);
}
let peer = stream.peer_addr().unwrap_or(addr);
let mut control = FrameTransport::new(stream);
control
.send(&json_frame(
FrameType::Handshake,
&Hello {
protocol: ATP_RQ_PROTOCOL,
role: "sender".to_string(),
peer_id: peer_id.to_string(),
symbol_size: config.symbol_size,
max_block_size: config.max_block_size as u64,
symbol_auth: symbol_auth_enabled,
total_bytes,
prefer_control_source_stream,
},
)?)
.await?;
let ack_frame = control.recv().await?;
if ack_frame.frame_type() != FrameType::HandshakeAck {
return Err(RqError::Unexpected {
got: ack_frame.frame_type(),
expected: "HandshakeAck",
});
}
let ack: HelloAck = parse_json(&ack_frame)?;
if !ack.accepted {
return Err(RqError::HandshakeRejected(
ack.reason.unwrap_or_else(|| "no reason given".to_string()),
));
}
if ack.control_source_stream && !prefer_control_source_stream {
return Err(RqError::HandshakeRejected(
"receiver selected control source stream for an ineligible transfer".to_string(),
));
}
let control_source_stream = ack.control_source_stream;
let udp_ports = if control_source_stream {
SmallVec::<[u16; DEFAULT_UDP_FANOUT]>::new()
} else {
hello_ack_udp_ports(&ack)
};
rqtrace!(
"sender: handshake ok, peer udp_ports={:?} control_source_stream={}",
udp_ports.as_slice(),
control_source_stream
);
let mut encoders: Vec<EntryEncoder> = Vec::with_capacity(entries.len());
for (i, (entry, digest)) in entries.iter().zip(digests.iter()).enumerate() {
let index = u32::try_from(i).unwrap_or(u32::MAX);
let size = usize::try_from(digest.size).map_err(|_| RqError::TooLarge {
size: digest.size,
max: u64::try_from(usize::MAX).unwrap_or(u64::MAX),
})?;
let source_offset =
usize::try_from(entry.source_offset).map_err(|_| RqError::TooLarge {
size: entry.source_offset,
max: u64::try_from(usize::MAX).unwrap_or(u64::MAX),
})?;
encoders.push(EntryEncoder {
index,
object_id: entry_object_id(&transfer_id, index),
abs_path: entry.abs_path.clone(),
source_offset,
size,
repair_cursors: Vec::new(),
});
}
// Send the manifest, then spray round 0 (source + optional overhead repair).
control
.send(&json_frame(FrameType::ObjectManifest, &manifest)?)
.await?;
if control_source_stream {
let digest_report = stream_control_source_entries(
cx,
&mut control,
&encoders,
&manifest,
&logical_digests,
&transfer_id,
symbol_auth.as_ref(),
)
.await?;
if digest_report.bytes_streamed != total_bytes {
return Err(RqError::Source(format!(
"control source stream sent {} bytes, expected {total_bytes}",
digest_report.bytes_streamed
)));
}
let merkle_root_hex = digest_report.merkle_root_hex.clone();
control
.send(&json_frame(
FrameType::ObjectComplete,
&RqRoundComplete {
round_symbols_sent: 0,
entry_digests: digest_report.entry_digests,
logical_digests: digest_report.logical_digests,
merkle_root_hex: Some(merkle_root_hex.clone()),
},
)?)
.await?;
let reply = control.recv().await?;
if reply.frame_type() != FrameType::Proof {
return Err(RqError::Unexpected {
got: reply.frame_type(),
expected: "Proof",
});
}
let receipt: ReceiveReceipt = parse_json(&reply)?;
let _ = control
.send(&Frame::empty(FrameType::Close).map_err(|e| RqError::Frame(e.to_string()))?)
.await;
if !receipt.committed {
return Err(RqError::Integrity(
receipt
.reason
.clone()
.unwrap_or_else(|| "receiver did not commit".to_string()),
));
}
return Ok(SendReport {
transfer_id,
bytes_sent: total_bytes,
files: u32::try_from(logical_digests.len()).unwrap_or(u32::MAX),
symbols_sent: 0,
feedback_rounds: 0,
merkle_root_hex,
receipt,
udp_send_acceleration: UdpSendAccelerationReport::default(),
peer,
});
}
// Data plane: open UDP sockets connected across the receiver's advertised UDP fanout.
let fanout = config.udp_fanout.max(1);
let mut adaptive = RqAdaptiveSendState::new(tag, &config, fanout);
let local_unspec = if peer.ip().is_ipv4() {
std::net::IpAddr::V4(std::net::Ipv4Addr::UNSPECIFIED)
} else {
std::net::IpAddr::V6(std::net::Ipv6Addr::UNSPECIFIED)
};
let mut sockets: Vec<UdpSocket> = Vec::with_capacity(fanout);
for socket_index in 0..fanout {
let udp_addr = receiver_udp_addr_for_socket(peer, &udp_ports, socket_index)?;
let sock = UdpSocket::bind(SocketAddr::new(local_unspec, 0)).await?;
sock.connect(udp_addr).await?;
// Large send buffer absorbs bursts so the spray loop does not busy-spin
// on `ENOBUFS`/`WouldBlock` (UDP sockets epoll-report writable even when
// the send buffer is full).
let _ = sock.tune_buffers(UdpBufferConfig {
send_buffer_bytes: Some(16 * 1024 * 1024),
recv_buffer_bytes: None,
});
sockets.push(sock);
}
let mut symbols_sent: u64 = 0;
let mut rr = 0usize;
let mut dropper = 0u32;
let mut feedback_rounds = 0u32;
let mut source_fec_fallback_active = false;
let mut udp_send_acceleration = UdpSendAccelerationReport::default();
// Parallel per-block encode is on by default while the transfer is small enough that the
// receiver's recv buffer can absorb the burst (see PARALLEL_ENCODE_MAX_BYTES). Larger
// loss-targeted transfers use a much smaller encode window: the pacer still owns wire rate,
// but the sender is no longer forced to mint every source+repair block on one core.
let parallel_encode_plan = parallel_encode_plan_for_transfer(total_bytes, &config);
rqtrace!(
"sender: encode_plan total_bytes={} entries={} parallel_encode_batch_blocks={}",
total_bytes,
manifest.entries.len(),
parallel_encode_plan
.map(|plan| plan.max_batch_blocks)
.unwrap_or(0),
);
// Round 0: every entry, source symbols plus optional repair_overhead extra.
let mut pending: BTreeSet<u32> = encoders.iter().map(|e| e.index).collect();
let round0_small_clean_source_only = small_clean_source_only_round0(total_bytes, &config);
let mut round_tuning =
apply_small_clean_round0_source_only(total_bytes, &config, adaptive.round0_tuning(&config));
if round0_small_clean_source_only {
rqtrace!(
"sender: round0_small_clean_source_only total_bytes={} repair_overhead={:.4}",
total_bytes,
round_tuning.repair_overhead
);
}
// One token bucket owns the whole transfer. Recreating it per source/repair
// spray would grant a fresh burst each time and overflow rate-capped qdiscs.
let mut pacer =
RqSprayPacer::new_round0(round_tuning.pacing, &config, round0_small_clean_source_only);
let mut round_started = Instant::now();
let mut round_symbols_start = symbols_sent;
let mut round_delivery_sample_kind = RqDeliverySampleKind::InitialOrRepair;
spray_round(
cx,
&mut control,
&mut adaptive,
&mut sockets,
&mut rr,
&mut symbols_sent,
&mut dropper,
tag,
&mut encoders,
&pending,
&config,
&mut pacer,
&round_tuning,
symbol_auth.as_ref(),
&mut udp_send_acceleration,
/* with_source */ true,
round0_small_clean_source_only,
parallel_encode_plan,
)
.await?;
let mut round_send_wall = round_started.elapsed();
rqtrace!("sender: round 0 sprayed, symbols_sent={symbols_sent}");
// Feedback loop.
let mut peak_sender_window_bytes = 0u64;
loop {
let control_wait_started = Instant::now();
let sent_this_round = symbols_sent.saturating_sub(round_symbols_start);
control
.send(&json_frame(
FrameType::ObjectComplete,
&RqRoundComplete {
round_symbols_sent: sent_this_round,
..RqRoundComplete::default()
},
)?)
.await?;
rqtrace!("sender: sent ObjectComplete, awaiting reply");
let reply = control.recv().await?;
let control_wait = control_wait_started.elapsed();
let window_probe = RqSenderWindowProbe::new(
pacer.pacing(),
sent_this_round,
config.symbol_size,
round_send_wall,
control_wait,
);
let window_probe_phase = match reply.frame_type() {
FrameType::Proof => "proof",
FrameType::ObjectRequest => "need_more",
FrameType::KeepAlive => "keep_alive",
_ => "other",
};
peak_sender_window_bytes = peak_sender_window_bytes.max(window_probe.peak_window_bytes());
trace_sender_window_probe(
window_probe_phase,
feedback_rounds,
window_probe,
peak_sender_window_bytes,
udp_send_acceleration,
);
rqtrace!("sender: got reply {:?}", reply.frame_type());
match reply.frame_type() {
FrameType::Proof => {
adaptive.observe_probe_success(
&config,
sent_this_round,
round_send_wall,
control_wait,
);
let receipt: ReceiveReceipt = parse_json(&reply)?;
let _ = control
.send(
&Frame::empty(FrameType::Close)
.map_err(|e| RqError::Frame(e.to_string()))?,
)
.await;
if !receipt.committed {
return Err(RqError::Integrity(
receipt
.reason
.clone()
.unwrap_or_else(|| "receiver did not commit".to_string()),
));
}
rqtrace!(
"sender: udp_send_acceleration flushes={} datagrams={} payload_bytes={} native_flushes={} native_datagrams={} gso_flushes={} gso_datagrams={} fallback_flushes={} fallback_datagrams={} partial_flushes={} error_flushes={}",
udp_send_acceleration.flushes,
udp_send_acceleration.datagrams,
udp_send_acceleration.payload_bytes,
udp_send_acceleration.native_batch_flushes,
udp_send_acceleration.native_batch_datagrams,
udp_send_acceleration.gso_flushes,
udp_send_acceleration.gso_datagrams,
udp_send_acceleration.fallback_flushes,
udp_send_acceleration.fallback_datagrams,
udp_send_acceleration.partial_flushes,
udp_send_acceleration.error_flushes,
);
return Ok(SendReport {
transfer_id,
bytes_sent: total_bytes,
files: u32::try_from(logical_digests.len()).unwrap_or(u32::MAX),
symbols_sent,
feedback_rounds,
merkle_root_hex,
receipt,
udp_send_acceleration,
peer,
});
}
FrameType::KeepAlive => {
adaptive.mark_control_peer_activity();
}
FrameType::ObjectRequest => {
let need: NeedMore = parse_json(&reply)?;
feedback_rounds += 1;
if feedback_rounds > config.max_feedback_rounds {
return Err(RqError::NoConvergence {
rounds: feedback_rounds,
pending: need.pending.len(),
});
}
let source_symbols = need.source_symbols;
let prior_pending_bytes = pending_bytes(&digests, &pending);
let progress = RqNeedMoreProgress {
pending_rank: need.pending_rank,
pending_rank_columns: need.pending_rank_columns,
pending_rank_deficit: need.pending_rank_deficit,
pending_decode_jobs: need.pending_decode_jobs,
};
pending = need.pending.into_iter().collect();
let fallback_received = sent_this_round.saturating_sub(
u64::try_from(pending.len())
.unwrap_or(u64::MAX)
.min(sent_this_round),
);
let received_this_round = need
.round_symbols_observed
.or(need.round_symbols_accepted)
.unwrap_or(fallback_received)
.min(sent_this_round);
let feedback_delivery_bps =
received_this_round.saturating_mul(u64::from(config.symbol_size.max(1))) as f64
/ finite_duration_s(round_send_wall);
if pending.is_empty() {
// Receiver says nothing pending but did not send Proof yet;
// loop again to fetch the Proof.
continue;
}
// Wire loss for pacing/AIMD comes from ARRIVALS when the
// receiver counted them: `round_symbols_observed` includes
// duplicates and rank-stalled symbols — everything that
// proved delivery. The receiver's `round_loss_fraction`
// folds in usefulness (post-completion excess reads as
// loss): on the 500M/broken cell it reported 0.59 while
// arrivals proved 0.092, halving the AIMD rate every round
// (MATRIX-207). Passing None here lets observe_need_more
// derive loss from the arrival count itself.
let pacing_round_loss_fraction = if need.round_symbols_observed.is_some() {
None
} else {
need.round_loss_fraction
};
adaptive.observe_need_more_with_progress(
&config,
&digests,
&pending,
prior_pending_bytes,
progress,
sent_this_round,
received_this_round,
pacing_round_loss_fraction,
round_delivery_sample_kind,
round_send_wall,
control_wait,
total_bytes,
);
let measured_repair_overhead =
measured_feedback_repair_overhead(adaptive.last_round_loss_fraction);
let source_fec_fallback_trigger = source_retransmit_needs_fec_fallback(
&config,
feedback_rounds,
source_symbols.len(),
adaptive.last_round_loss_fraction,
);
source_fec_fallback_active |= source_fec_fallback_trigger;
round_tuning = if source_fec_fallback_active {
adaptive.source_fec_fallback_tuning(&config)
} else {
adaptive.round_tuning(&config)
};
pacer.configure_with_shared_decision(
round_tuning.pacing,
adaptive.shared_rate_decision(),
);
let loss_pacing_cap_bps = adaptive.loss_pacing_cap_bps.unwrap_or(0);
rqtrace!(
"sender: NeedMore round={feedback_rounds} pending={} source_requests={} sent_this_round={} received_this_round={} round_loss_fraction={:.4} measured_repair_overhead={:.4} fallback_trigger={} aimd_rate_bps={} loss_pacing_cap_bps={} send_wall_ms={} control_wait_ms={} delivery_rate_bps={:.0} bw_ema_bps={:.0} bw_trough_bps={:.0} repair_overhead={:.4} path_rate_bps={} repair_loss_ema={:.4} pacing_loss_ema={:.4} repair_loss_bar={:.4} pacing_loss_bar={:.4} fec_fallback={}",
pending.len(),
source_symbols.len(),
sent_this_round,
received_this_round,
adaptive.last_round_loss_fraction,
measured_repair_overhead,
source_fec_fallback_trigger,
adaptive.aimd_rate_bps,
loss_pacing_cap_bps,
round_send_wall.as_millis(),
control_wait.as_millis(),
feedback_delivery_bps,
adaptive.bw_ema_bps,
adaptive.bw_trough_bps,
round_tuning.repair_overhead,
round_tuning.pacing.path_rate_bps,
adaptive.loss_ema,
adaptive.pacing_loss_ema,
adaptive.loss_bar,
adaptive.pacing_loss_bar,
source_fec_fallback_active,
);
// A request list below the cap enumerates the ENTIRE residual
// (collect_source_requests truncates at the limit): the
// requested systematic symbols are exactly what the remaining
// blocks are missing, so the blanket per-block fallback spray
// would land almost entirely on already-complete blocks
// (99.4% of a 78MB spray round rejected on 500M/broken,
// MATRIX-207). Only spray when the list was truncated and the
// true residual is unknown.
let residual_fully_requested = !source_symbols.is_empty()
&& (config.max_source_retransmit_requests == 0
|| source_symbols.len() < config.max_source_retransmit_requests);
let spray_fallback = source_fec_fallback_active && !residual_fully_requested;
if source_symbols.is_empty() {
round_started = Instant::now();
round_symbols_start = symbols_sent;
// Fresh repair symbols (true encoder ESIs, via the
// cumulative cursor in each EntryEncoder) for the
// still-pending entries.
spray_round(
cx,
&mut control,
&mut adaptive,
&mut sockets,
&mut rr,
&mut symbols_sent,
&mut dropper,
tag,
&mut encoders,
&pending,
&config,
&mut pacer,
&round_tuning,
symbol_auth.as_ref(),
&mut udp_send_acceleration,
/* with_source */ false,
false,
parallel_encode_plan,
)
.await?;
round_send_wall = round_started.elapsed();
round_delivery_sample_kind = RqDeliverySampleKind::InitialOrRepair;
} else {
round_started = Instant::now();
round_symbols_start = symbols_sent;
spray_source_requests(
cx,
&mut control,
&mut adaptive,
&mut sockets,
&mut rr,
&mut symbols_sent,
&mut dropper,
tag,
&encoders,
&source_symbols,
&config,
&mut pacer,
symbol_auth.as_ref(),
&mut udp_send_acceleration,
)
.await?;
if spray_fallback {
spray_round(
cx,
&mut control,
&mut adaptive,
&mut sockets,
&mut rr,
&mut symbols_sent,
&mut dropper,
tag,
&mut encoders,
&pending,
&config,
&mut pacer,
&round_tuning,
symbol_auth.as_ref(),
&mut udp_send_acceleration,
/* with_source */ false,
false,
parallel_encode_plan,
)
.await?;
}
round_send_wall = round_started.elapsed();
round_delivery_sample_kind = delivery_sample_kind_for_need_more_response(
source_symbols.len(),
spray_fallback,
);
}
let emitted_this_response = symbols_sent.saturating_sub(round_symbols_start);
rqtrace!(
"sender: NeedMore response round={feedback_rounds} pending={} source_requests={} emitted_symbols={} total_symbols_sent={} max_feedback_rounds={} repair_overhead={:.4} fec_fallback={}",
pending.len(),
source_symbols.len(),
emitted_this_response,
symbols_sent,
config.max_feedback_rounds,
round_tuning.repair_overhead,
spray_fallback,
);
}
other => {
return Err(RqError::Unexpected {
got: other,
expected: "Proof | NeedMore | KeepAlive",
});
}
}
}
}
/// Legacy ceiling for per-round repair increments.
///
/// This preserves the old high-loss convergence cap while letting clean-link
/// feedback rounds avoid spraying a fixed K/4 parity burst for a sparse tail.
fn max_feedback_repair_batch_per_block(block_source_n: usize) -> usize {
(block_source_n / 4).max(16)
}
/// Minimum fresh repair symbols emitted for a source-first repair-only feedback
/// round. A one-symbol tail is too fragile on lossy links: if that one repair
/// symbol drops, the receiver idles until the next control PTO or fails closed.
const SOURCE_FIRST_FEEDBACK_REPAIR_FLOOR_PER_BLOCK: usize = 4;
fn adaptive_feedback_repair_batch_per_block(block_source_n: usize, repair_overhead: f64) -> usize {
if repair_overhead <= 1.0 {
return SOURCE_FIRST_FEEDBACK_REPAIR_FLOOR_PER_BLOCK;
}
let matched = ((block_source_n as f64) * (repair_overhead - 1.0)).ceil() as usize;
matched
.max(SOURCE_FIRST_FEEDBACK_REPAIR_FLOOR_PER_BLOCK)
.min(max_feedback_repair_batch_per_block(block_source_n))
}
fn initial_repair_target_per_block(block_source_n: usize, repair_overhead: f64) -> usize {
if repair_overhead <= 1.0 {
0
} else {
((block_source_n as f64) * (repair_overhead - 1.0)).ceil() as usize
}
}
fn repair_target_for_feedback_round(
block_source_n: usize,
already: usize,
repair_overhead: f64,
) -> usize {
let calibrated_total = initial_repair_target_per_block(block_source_n, repair_overhead);
if calibrated_total > already {
calibrated_total
} else {
already + adaptive_feedback_repair_batch_per_block(block_source_n, repair_overhead)
}
}
fn source_retransmit_request_limit(config: &RqConfig, feedback_round: u32) -> Option<usize> {
// Loss-target cells: round-0 FEC carries the bulk of the transfer, so by
// the first feedback round most blocks have decoded and the residual is a
// FEW rank-deficient blocks. Sparse systematic requests pinpoint exactly
// those symbols; the blanket per-block fallback spray cannot (on the
// 500M/broken cell 99.4% of a 78MB spray round landed on already-complete
// blocks and was rejected, MATRIX-207). No round cap: the residual only
// shrinks, so requests stay the cheapest mechanism every round.
if round0_loss_target_repair_enabled(config) {
return Some(config.max_source_retransmit_requests);
}
if config.repair_overhead <= 1.0
&& config.source_retransmit_rounds > 0
&& feedback_round <= config.source_retransmit_rounds
{
Some(config.max_source_retransmit_requests)
} else {
None
}
}
fn source_retransmit_needs_fec_fallback(
config: &RqConfig,
feedback_round: u32,
requested_sources: usize,
measured_loss_fraction: f64,
) -> bool {
if measured_feedback_repair_overhead(measured_loss_fraction) > 0.0 {
return true;
}
if round0_loss_target_repair_enabled(config) {
return true;
}
if config.repair_overhead > 1.0 || config.source_retransmit_rounds == 0 {
return false;
}
let rank_only_or_repair_feedback = requested_sources == 0;
let saturated_request = config.max_source_retransmit_requests != 0
&& requested_sources >= config.max_source_retransmit_requests;
rank_only_or_repair_feedback
|| saturated_request
|| feedback_round >= config.source_retransmit_rounds
}
/// Above this total transfer size the parallel per-block encode is disabled and the sequential
/// (encode-paced) spray is used instead. The receiver sizes its UDP recv buffer to absorb the
/// parallel sender's burst, but that buffer is clamped near `net.core.rmem_max`; once a transfer
/// exceeds what the buffer can hold, an unpaced parallel burst would overrun the CPU-bound decoder
/// and trigger a feedback-round explosion. Below the cap the burst is absorbed and the encode
/// parallelism is a pure win. Parallel decode + a rate-paced encode-ahead ring are what lift this
/// cap for very large objects.
const PARALLEL_ENCODE_MAX_BYTES: u64 = 112 * 1024 * 1024;
const PARALLEL_ENCODE_HOST_MAX_BATCH_BLOCKS: usize = 64;
/// Lossy objects need encode parallelism to keep the 50 mbit bad link fed, but
/// they must not recreate the old full-transfer burst. Eight bounded RaptorQ
/// blocks is enough CPU fanout for the target link while keeping peak sender
/// symbol RAM well below the receiver's UDP buffer envelope.
const LOSSY_LARGE_PARALLEL_ENCODE_BATCH_BLOCKS: usize = 8;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
struct ParallelEncodePlan {
max_batch_blocks: usize,
}
fn parallel_encode_plan_for_transfer(
total_bytes: u64,
config: &RqConfig,
) -> Option<ParallelEncodePlan> {
if round0_loss_target_repair_enabled(config) {
Some(ParallelEncodePlan {
max_batch_blocks: LOSSY_LARGE_PARALLEL_ENCODE_BATCH_BLOCKS,
})
} else if total_bytes <= PARALLEL_ENCODE_MAX_BYTES {
Some(ParallelEncodePlan {
max_batch_blocks: PARALLEL_ENCODE_HOST_MAX_BATCH_BLOCKS,
})
} else {
None
}
}
fn parallel_encode_window_blocks(plan: ParallelEncodePlan) -> usize {
let host_batch = std::thread::available_parallelism()
.map_or(4, std::num::NonZeroUsize::get)
.clamp(2, PARALLEL_ENCODE_HOST_MAX_BATCH_BLOCKS);
host_batch.min(
plan.max_batch_blocks
.clamp(2, PARALLEL_ENCODE_HOST_MAX_BATCH_BLOCKS),
)
}
/// Upper bound on the number of source blocks we fan out across the blocking pool in one round.
/// Above this the manual block enumeration would risk diverging from the canonical encoder's `u8`
/// SBN envelope, so we fall back to the sequential encode-paced spray.
const MAX_RAPTORQ_SOURCE_BLOCKS: usize = 256;
/// Whether a round-0 (`with_source`) spray should fan its per-block RaptorQ solves out across the
/// runtime blocking pool. We parallelize only multi-block objects (a single/empty block would only
/// pay pool-dispatch latency and lose the small-object latency win), only while `parallel_encode`
/// is on (the transfer fits under [`PARALLEL_ENCODE_MAX_BYTES`]), and only within the `u8` SBN
/// envelope.
fn should_parallel_encode_source_blocks(
block_count: usize,
parallel_encode_plan: Option<ParallelEncodePlan>,
) -> bool {
parallel_encode_plan.is_some() && block_count > 1 && block_count <= MAX_RAPTORQ_SOURCE_BLOCKS
}
/// Encode one RaptorQ source block (its `K` source symbols plus `repair_count` repair symbols) into
/// an owned `Vec<Symbol>`.
///
/// Runs on the blocking pool for [`spray_round`]'s parallel per-block encode.
/// [`EncodingPipeline::encode_single_block_with_repair`] preserves the exact object/SBN/ESI layout
/// the sequential per-block path would have produced for `sbn`, so the emitted symbols are
/// byte-identical regardless of which thread minted them — the speedup is a pure throughput
/// isomorphism (decode is order-independent and the receiver verifies sha256 + merkle). The error
/// is stringified because the closure crosses the `spawn_blocking` boundary, where the return type
/// need only be `Send`.
fn encode_block_symbols(
cfg: &crate::config::EncodingConfig,
object_id: ObjectId,
sbn: u8,
data: &[u8],
repair_count: usize,
) -> Result<Vec<Symbol>, String> {
let pool = SymbolPool::new(PoolConfig::default());
let mut pipeline = EncodingPipeline::new(cfg.clone(), pool);
let mut out = Vec::new();
for encoded in pipeline.encode_single_block_with_repair(object_id, sbn, data, repair_count) {
out.push(encoded.map_err(|e| e.to_string())?.into_symbol());
}
Ok(out)
}
fn source_symbol_from_block(
object_id: ObjectId,
sbn: u8,
esi: usize,
block_bytes: &[u8],
symbol_size: usize,
) -> Result<Symbol, RqError> {
let within_block = esi
.checked_mul(symbol_size)
.ok_or_else(|| RqError::Coding("source symbol offset overflow".to_string()))?;
let esi = u32::try_from(esi)
.map_err(|_| RqError::Coding(format!("source symbol ESI {esi} exceeds u32::MAX")))?;
let mut payload = vec![0u8; symbol_size];
if within_block < block_bytes.len() {
let end = within_block
.saturating_add(symbol_size)
.min(block_bytes.len());
payload[..end - within_block].copy_from_slice(&block_bytes[within_block..end]);
}
Ok(Symbol::new(
SymbolId::new(object_id, sbn, esi),
payload,
SymbolKind::Source,
))
}
#[allow(clippy::too_many_arguments)]
async fn send_source_only_block_datagrams<S>(
cx: &Cx,
control: &mut FrameTransport<S>,
adaptive: &mut RqAdaptiveSendState,
sockets: &mut [UdpSocket],
rr: &mut usize,
symbols_sent: &mut u64,
dropper: &mut u32,
tag: u64,
entry: u32,
object_id: ObjectId,
block: EncodeAheadBlock,
block_bytes: &[u8],
config: &RqConfig,
pacer: &mut RqSprayPacer,
symbol_auth: Option<&SecurityContext>,
udp_send_acceleration: &mut UdpSendAccelerationReport,
) -> Result<usize, RqError>
where
S: AsyncReadExt + AsyncWriteExt + Unpin,
{
let symbol_size = usize::from(config.symbol_size.max(1));
let mut send_batch = RqPendingSendBatch::new(sockets.len());
for esi in 0..block.k {
let symbol = source_symbol_from_block(object_id, block.sbn, esi, block_bytes, symbol_size)?;
queue_symbol_datagram(
cx,
control,
adaptive,
sockets,
rr,
symbols_sent,
dropper,
tag,
entry,
&symbol,
config,
pacer,
symbol_auth,
&mut send_batch,
udp_send_acceleration,
)
.await?;
}
let report = send_batch.flush(sockets, symbols_sent).await?;
udp_send_acceleration.observe_flush_report(report);
service_rq_spray_control(cx, control, adaptive).await?;
Ok(block.k)
}
/// Spray one round of symbols for the `pending` entries across the UDP sockets.
///
/// Round 0 (`with_source`) sends every block's source symbols plus optional
/// `repair_overhead` extra repair. Feedback rounds send only *newly minted*
/// repair symbols, identified per block by the encoder's own (sbn, esi) — the
/// repair payload is bound to its ESI, so it is emitted verbatim and never
/// relabeled. Per-block repair cursors advance so each round's repair is fresh
/// for every source block in a pending entry.
#[allow(clippy::too_many_arguments)]
async fn spray_round<S>(
cx: &Cx,
control: &mut FrameTransport<S>,
adaptive: &mut RqAdaptiveSendState,
sockets: &mut [UdpSocket],
rr: &mut usize,
symbols_sent: &mut u64,
dropper: &mut u32,
tag: u64,
encoders: &mut [EntryEncoder],
pending: &BTreeSet<u32>,
config: &RqConfig,
pacer: &mut RqSprayPacer,
round_tuning: &RqRoundTuning,
symbol_auth: Option<&SecurityContext>,
udp_send_acceleration: &mut UdpSendAccelerationReport,
with_source: bool,
small_clean_source_only: bool,
parallel_encode_plan: Option<ParallelEncodePlan>,
) -> Result<(), RqError>
where
S: AsyncReadExt + AsyncWriteExt + Unpin,
{
for enc in encoders.iter_mut().filter(|e| pending.contains(&e.index)) {
cx.checkpoint().map_err(|_| RqError::Cancelled)?;
let mut ring = EncodeAheadRing::default();
let blocks = encode_ahead_blocks(enc.size, config)?;
if enc.repair_cursors.len() > blocks.len() {
enc.repair_cursors.truncate(blocks.len());
}
if enc.repair_cursors.len() < blocks.len() {
enc.repair_cursors.resize(blocks.len(), 0);
}
let mut round0_blocks = 0usize;
let mut round0_source_symbols = 0usize;
let mut round0_repair_symbols = 0usize;
let use_parallel_source_encode =
with_source && should_parallel_encode_source_blocks(blocks.len(), parallel_encode_plan);
if with_source && small_clean_source_only {
for (block_index, block) in blocks.iter().copied().enumerate() {
let read_start = enc.source_offset.checked_add(block.start).ok_or_else(|| {
RqError::Coding("encode source range offset overflow".to_string())
})?;
let block_bytes = read_source_range(&enc.abs_path, read_start, block.len).await?;
let source_symbols = send_source_only_block_datagrams(
cx,
control,
adaptive,
sockets,
rr,
symbols_sent,
dropper,
tag,
enc.index,
enc.object_id,
block,
&block_bytes,
config,
pacer,
symbol_auth,
udp_send_acceleration,
)
.await?;
round0_blocks = round0_blocks.saturating_add(1);
round0_source_symbols = round0_source_symbols.saturating_add(source_symbols);
enc.repair_cursors[block_index] = 0;
}
} else if use_parallel_source_encode {
// Parallel per-block encode on the runtime blocking pool. Each RaptorQ source block
// solves independently, so for multi-block objects we fan the K-symbol solves across
// cores instead of grinding them one-at-a-time on a single core (the measured
// large-file bottleneck: ~99% of one core for an 8 MiB / K=8192 block). Blocks are
// encoded and sprayed in SBN order, so the wire output is byte-identical to the
// sequential path — a pure throughput isomorphism (decode is order-independent; the
// receiver verifies sha256 + merkle). Bounded BATCHES (degree = host parallelism) cap
// peak symbol RAM at ~2x `par_batch` blocks (double-buffered below); the in-flight
// window is joined on the checkpoint-cancel path so no encode task strands.
let enc_cfg = crate::config::EncodingConfig {
repair_overhead: round_tuning.repair_overhead,
max_block_size: config.max_block_size,
symbol_size: config.symbol_size,
encoding_parallelism: 1,
decoding_parallelism: 1,
};
let par_batch = parallel_encode_window_blocks(
parallel_encode_plan.expect("parallel encode plan checked above"),
);
// Double-buffered encode-ahead: spawn window W+1's encodes BEFORE
// draining window W, so the pool encodes the next window while the
// pacer is emitting the current one. The serial spawn-join-spawn
// shape stalled the paced spray ~300-400 ms per window (encode
// latency un-overlapped with the ~4 s paced send), a ~7% realized
// round-0 rate loss on the 10 mbit broken cell (MATRIX-209).
// Steady state holds at most TWO windows of symbols; the pending
// window is drained on the checkpoint-cancel path, and hard-error
// unwinds drain via region close (quiescence).
let mut pending: Vec<(
usize,
usize,
usize,
crate::runtime::TaskHandle<Result<Vec<Symbol>, String>>,
)> = Vec::new();
for window_start in (0..blocks.len()).step_by(par_batch) {
if cx.checkpoint().is_err() {
for (_, _, _, mut handle) in pending.drain(..) {
let _ = handle.join(cx).await;
}
return Err(RqError::Cancelled);
}
let window_end = (window_start + par_batch).min(blocks.len());
let window = &blocks[window_start..window_end];
let mut spawned = Vec::with_capacity(window.len());
for (window_offset, block) in window.iter().enumerate() {
// Disk reads are cheap relative to the RaptorQ solve, so read each block's
// source range here and hand the owned bytes to the pool task.
let read_start =
enc.source_offset.checked_add(block.start).ok_or_else(|| {
RqError::Coding("encode source range offset overflow".to_string())
})?;
let block_bytes =
read_source_range(&enc.abs_path, read_start, block.len).await?;
let object_id = enc.object_id;
let sbn = block.sbn;
let block_index = window_start + window_offset;
let repair =
initial_repair_target_per_block(block.k, round_tuning.repair_overhead);
let cfg = enc_cfg.clone();
let handle = cx
.spawn_blocking(move |_child| {
encode_block_symbols(&cfg, object_id, sbn, &block_bytes, repair)
})
.map_err(|e| RqError::Coding(format!("encode spawn failed: {e:?}")))?;
spawned.push((block_index, block.k, repair, handle));
}
for (block_index, source_symbols, target_repair, mut handle) in pending.drain(..) {
let syms = match handle.join(cx).await {
Ok(Ok(syms)) => syms,
Ok(Err(e)) => return Err(RqError::Coding(e)),
Err(join_err) => {
return Err(RqError::Coding(format!(
"encode task failed: {join_err:?}"
)));
}
};
send_symbol_datagrams(
cx,
control,
adaptive,
sockets,
rr,
symbols_sent,
dropper,
tag,
enc.index,
&syms,
config,
pacer,
symbol_auth,
udp_send_acceleration,
)
.await?;
round0_blocks = round0_blocks.saturating_add(1);
round0_source_symbols = round0_source_symbols.saturating_add(source_symbols);
round0_repair_symbols = round0_repair_symbols.saturating_add(target_repair);
enc.repair_cursors[block_index] = target_repair;
}
pending = spawned;
}
for (block_index, source_symbols, target_repair, mut handle) in pending {
let syms = match handle.join(cx).await {
Ok(Ok(syms)) => syms,
Ok(Err(e)) => return Err(RqError::Coding(e)),
Err(join_err) => {
return Err(RqError::Coding(format!("encode task failed: {join_err:?}")));
}
};
send_symbol_datagrams(
cx,
control,
adaptive,
sockets,
rr,
symbols_sent,
dropper,
tag,
enc.index,
&syms,
config,
pacer,
symbol_auth,
udp_send_acceleration,
)
.await?;
round0_blocks = round0_blocks.saturating_add(1);
round0_source_symbols = round0_source_symbols.saturating_add(source_symbols);
round0_repair_symbols = round0_repair_symbols.saturating_add(target_repair);
enc.repair_cursors[block_index] = target_repair;
}
} else {
let mut feedback_repair_blocks = 0usize;
let mut feedback_source_symbols = 0usize;
let mut feedback_repair_symbols = 0usize;
let mut feedback_prior_repair_cursor = 0usize;
let mut feedback_target_repair_cursor = 0usize;
for (block_index, block) in blocks.iter().enumerate() {
// Cumulative repair count requested from the encoder for this
// block. The encoder always yields repair symbols at
// deterministic ESIs starting at the block's K'; requesting
// more just extends the tail. We skip the already-sent repair
// symbols for this block and emit the rest at their TRUE ESIs.
let already = enc.repair_cursors[block_index];
let target_repair = if with_source {
initial_repair_target_per_block(block.k, round_tuning.repair_overhead)
} else {
repair_target_for_feedback_round(block.k, already, round_tuning.repair_overhead)
};
let repair_count = target_repair.saturating_sub(already);
if with_source {
round0_blocks = round0_blocks.saturating_add(1);
round0_source_symbols = round0_source_symbols.saturating_add(block.k);
round0_repair_symbols = round0_repair_symbols.saturating_add(target_repair);
}
if !with_source {
feedback_repair_blocks += 1;
feedback_source_symbols = feedback_source_symbols.saturating_add(block.k);
feedback_repair_symbols = feedback_repair_symbols.saturating_add(repair_count);
feedback_prior_repair_cursor =
feedback_prior_repair_cursor.saturating_add(already);
feedback_target_repair_cursor =
feedback_target_repair_cursor.saturating_add(target_repair);
}
if !with_source && repair_count == 0 {
enc.repair_cursors[block_index] = target_repair;
continue;
}
// The encoder's `Symbol` output owns its payload buffer, so buffers
// allocated from `SymbolPool` are consumed rather than returned to
// the pool. Keep the M=1 encode-ahead path unpooled; round sizing,
// UDP pacing, and receiver-side limits own memory pressure.
let pool = SymbolPool::new(PoolConfig::default());
let mut pipeline = EncodingPipeline::new(
crate::config::EncodingConfig {
repair_overhead: round_tuning.repair_overhead,
max_block_size: config.max_block_size,
symbol_size: config.symbol_size,
encoding_parallelism: 1,
decoding_parallelism: 1,
},
pool,
);
let read_start = enc.source_offset.checked_add(block.start).ok_or_else(|| {
RqError::Coding("encode source range offset overflow".to_string())
})?;
let block_bytes = read_source_range(&enc.abs_path, read_start, block.len).await?;
let mut send_batch = RqPendingSendBatch::new(sockets.len());
if with_source {
for encoded in pipeline.encode_single_block_with_repair(
enc.object_id,
block.sbn,
&block_bytes,
target_repair,
) {
let encoded = encoded.map_err(|e| RqError::Coding(e.to_string()))?;
ring.push(EncodeAheadSymbol::from_encoded(enc.index, encoded))?;
let produced = ring.pop().expect("M=1 ring drains immediately");
queue_symbol_datagram(
cx,
control,
adaptive,
sockets,
rr,
symbols_sent,
dropper,
tag,
produced.entry,
&produced.symbol,
config,
pacer,
symbol_auth,
&mut send_batch,
udp_send_acceleration,
)
.await?;
debug_assert!(ring.is_empty());
}
} else {
for encoded in pipeline.encode_single_block_repair_range(
enc.object_id,
block.sbn,
&block_bytes,
already,
repair_count,
) {
let encoded = encoded.map_err(|e| RqError::Coding(e.to_string()))?;
ring.push(EncodeAheadSymbol::from_encoded(enc.index, encoded))?;
let produced = ring.pop().expect("M=1 ring drains immediately");
queue_symbol_datagram(
cx,
control,
adaptive,
sockets,
rr,
symbols_sent,
dropper,
tag,
produced.entry,
&produced.symbol,
config,
pacer,
symbol_auth,
&mut send_batch,
udp_send_acceleration,
)
.await?;
debug_assert!(ring.is_empty());
}
}
let report = send_batch.flush(sockets, symbols_sent).await?;
udp_send_acceleration.observe_flush_report(report);
service_rq_spray_control(cx, control, adaptive).await?;
enc.repair_cursors[block_index] = target_repair;
}
if !with_source {
let source_symbols = feedback_source_symbols.max(1) as f64;
let emitted_ratio = feedback_repair_symbols as f64 / source_symbols;
let target_ratio = feedback_target_repair_cursor as f64 / source_symbols;
rqtrace!(
"sender: repair_spray entry={} blocks={} source_symbols={} repair_overhead={:.4} emitted_repair_symbols={} emitted_repair_ratio={:.4} prior_repair_cursor={} target_repair_cursor={} target_repair_ratio={:.4} pending_entries={}",
enc.index,
feedback_repair_blocks,
feedback_source_symbols,
round_tuning.repair_overhead,
feedback_repair_symbols,
emitted_ratio,
feedback_prior_repair_cursor,
feedback_target_repair_cursor,
target_ratio,
pending.len(),
);
}
}
if with_source {
let source_symbols = round0_source_symbols.max(1) as f64;
let emitted_repair_ratio = round0_repair_symbols as f64 / source_symbols;
let emitted_symbols = round0_source_symbols.saturating_add(round0_repair_symbols);
rqtrace!(
"sender: round0_source_spray entry={} blocks={} source_symbols={} repair_overhead={:.4} emitted_repair_symbols={} emitted_repair_ratio={:.4} emitted_symbols={} pacing_rate_Bps={} pending_entries={}",
enc.index,
round0_blocks,
round0_source_symbols,
round_tuning.repair_overhead,
round0_repair_symbols,
emitted_repair_ratio,
emitted_symbols,
pacer.pacing().rate_bytes_per_sec(),
pending.len(),
);
}
}
Ok(())
}
fn apply_bonded_descriptor_config(
descriptor: &BondTransferDescriptor,
config: &mut RqConfig,
) -> Result<(), RqError> {
if descriptor.symbol_size == 0 {
return Err(RqError::Coding(
"bonded donor descriptor has zero symbol_size".to_string(),
));
}
let max_block_size =
usize::try_from(descriptor.max_block_size).map_err(|_| RqError::TooLarge {
size: descriptor.max_block_size,
max: u64::try_from(usize::MAX).unwrap_or(u64::MAX),
})?;
if max_block_size == 0 {
return Err(RqError::Coding(
"bonded donor descriptor has zero max_block_size".to_string(),
));
}
config.symbol_size = descriptor.symbol_size;
config.max_block_size = max_block_size;
Ok(())
}
fn bonded_initial_repair_symbols_per_block(config: &RqConfig) -> Result<u32, RqError> {
let tuning = RqAdaptiveSendState::new(0, config, 1).round0_tuning(config);
let block_source_n = usize::try_from(fixed_block_k(config)).map_err(|_| {
RqError::Coding(format!(
"bonded donor fixed block size does not fit usize: {}",
fixed_block_k(config)
))
})?;
let repair = initial_repair_target_per_block(block_source_n, tuning.repair_overhead);
u32::try_from(repair)
.map_err(|_| RqError::Coding(format!("bonded donor repair budget too large: {repair}")))
}
fn bonded_receiver_endpoints(
assignment: &DonorAssignment,
receiver_endpoint: SocketAddr,
) -> Vec<SocketAddr> {
let mut endpoints = Vec::with_capacity(assignment.receiver_udp_endpoints.len().max(1));
endpoints.push(receiver_endpoint);
for endpoint in &assignment.receiver_udp_endpoints {
if !endpoints.contains(endpoint) {
endpoints.push(*endpoint);
}
}
endpoints
}
fn bonded_donor_entry_path(root_dir: &Path, rel_path: &str) -> Result<PathBuf, RqError> {
if Path::new(rel_path).is_absolute() {
return Err(RqError::Source(format!(
"bonded donor rel_path escapes root: {rel_path}"
)));
}
let mut out = root_dir.to_path_buf();
let mut pushed = false;
for component in rel_path.split('/') {
if component.is_empty() || component == "." {
continue;
}
if component == ".." || Path::new(component).components().count() != 1 {
return Err(RqError::Source(format!(
"bonded donor rel_path escapes root: {rel_path}"
)));
}
out.push(component);
pushed = true;
}
if pushed {
Ok(out)
} else {
Err(RqError::Source(
"bonded donor rel_path must name a file".to_string(),
))
}
}
fn encode_bonded_donor_emission(
emission: BondedDonorSymbolEmission,
block_bytes: &[u8],
config: &RqConfig,
) -> Result<Symbol, RqError> {
let k = u32::from(emission.geometry.source_symbols);
let symbol = if emission.esi < k {
source_symbol_from_block(
emission.geometry.object_id,
emission.geometry.source_block_number,
usize::try_from(emission.esi).map_err(|_| {
RqError::Coding(format!(
"bonded donor source ESI does not fit usize: {}",
emission.esi
))
})?,
block_bytes,
usize::from(config.symbol_size.max(1)),
)?
} else {
let first_repair = usize::try_from(emission.esi - k).map_err(|_| {
RqError::Coding(format!(
"bonded donor repair ESI does not fit usize: {}",
emission.esi
))
})?;
let mut pipeline = EncodingPipeline::new(
crate::config::EncodingConfig {
repair_overhead: config.repair_overhead,
max_block_size: config.max_block_size,
symbol_size: config.symbol_size,
encoding_parallelism: 1,
decoding_parallelism: 1,
},
SymbolPool::new(PoolConfig::default()),
);
let mut encoded = pipeline.encode_single_block_repair_range(
emission.geometry.object_id,
emission.geometry.source_block_number,
block_bytes,
first_repair,
1,
);
let encoded = encoded
.next()
.ok_or_else(|| {
RqError::Coding(format!(
"bonded donor encoder produced no repair symbol for esi {}",
emission.esi
))
})?
.map_err(|err| RqError::Coding(err.to_string()))?;
encoded.symbol().clone()
};
if symbol.id() != emission.symbol_id() || symbol.kind() != emission.symbol_kind() {
return Err(RqError::Coding(format!(
"bonded donor encoded wrong symbol: expected sbn={} esi={} kind={:?}, got sbn={} esi={} kind={:?}",
emission.geometry.source_block_number,
emission.esi,
emission.symbol_kind(),
symbol.id().sbn(),
symbol.id().esi(),
symbol.kind(),
)));
}
Ok(symbol)
}
#[allow(clippy::too_many_arguments)]
async fn queue_bonded_donor_datagram(
cx: &Cx,
sockets: &mut [UdpSocket],
rr: &mut usize,
symbols_sent: &mut u64,
dropper: &mut u32,
tag: u64,
entry: u32,
sym: &Symbol,
config: &RqConfig,
pacer: &mut RqSprayPacer,
symbol_auth: Option<&SecurityContext>,
send_batch: &mut RqPendingSendBatch,
udp_send_acceleration: &mut UdpSendAccelerationReport,
) -> Result<(), RqError> {
cx.checkpoint().map_err(|_| RqError::Cancelled)?;
if config.debug_drop_one_in > 0 {
*dropper = dropper.wrapping_add(1);
if *dropper % config.debug_drop_one_in == 0 {
return Ok(());
}
}
pacer.before_send(cx).await?;
let auth = symbol_auth.map(|ctx| ctx.sign_symbol(sym));
let dgram =
encode_symbol_datagram(tag, entry, sym, auth.as_ref().map(AuthenticatedSymbol::tag));
let fanout = send_batch.fanout();
let socket_index = *rr % fanout;
*rr = rr.wrapping_add(1);
send_batch.push(socket_index, dgram);
pacer.observe_datagram_sent();
if send_batch.should_flush() {
let report = send_batch.flush(sockets, symbols_sent).await?;
udp_send_acceleration.observe_flush_report(report);
}
Ok(())
}
async fn hash_source_entry_streaming(
entry: &RqSourceEntry,
buf: &mut [u8],
) -> Result<(u64, crate::atp::object::ObjectId, [u8; 32]), StreamingError> {
if entry.source_offset == 0 && entry.source_len.is_none() {
return hash_file_streaming(&entry.abs_path, buf).await;
}
let len = entry.source_len.ok_or_else(|| {
StreamingError::new(format!(
"{}: ranged source entry missing source_len",
entry.abs_path.display()
))
})?;
hash_file_range_streaming(&entry.abs_path, entry.source_offset, len, buf).await
}
async fn hash_file_range_streaming(
path: &Path,
offset: u64,
len: u64,
buf: &mut [u8],
) -> Result<(u64, crate::atp::object::ObjectId, [u8; 32]), StreamingError> {
let mut file = crate::fs::File::open(path)
.await
.map_err(|e| StreamingError::new(format!("{}: {e}", path.display())))?;
file.seek(std::io::SeekFrom::Start(offset))
.await
.map_err(|e| StreamingError::new(format!("{}: {e}", path.display())))?;
let mut sha = Sha256::new();
let mut cid = crate::atp::object::ContentId::streaming();
let mut remaining = len;
let mut size = 0u64;
while remaining > 0 {
let want = usize::try_from(remaining.min(buf.len() as u64)).unwrap_or(buf.len());
let n = file
.read(&mut buf[..want])
.await
.map_err(|e| StreamingError::new(format!("{}: {e}", path.display())))?;
if n == 0 {
return Err(StreamingError::new(format!(
"{}: short read while hashing source range offset={offset} len={len}",
path.display()
)));
}
sha.update(&buf[..n]);
cid.update(&buf[..n]);
let n_u64 = n as u64;
remaining -= n_u64;
size = size.saturating_add(n_u64);
}
Ok((
size,
crate::atp::object::ObjectId::content(cid.finalize()),
sha.finalize().into(),
))
}
async fn read_source_range(path: &Path, offset: usize, len: usize) -> Result<Vec<u8>, RqError> {
if len == 0 {
return Ok(Vec::new());
}
let offset_u64 = u64::try_from(offset).map_err(|_| {
RqError::Coding(format!(
"{}: source range offset does not fit u64: {offset}",
path.display()
))
})?;
let mut file = crate::fs::File::open(path)
.await
.map_err(|e| RqError::Source(format!("{}: {e}", path.display())))?;
file.seek(std::io::SeekFrom::Start(offset_u64))
.await
.map_err(|e| RqError::Source(format!("{}: {e}", path.display())))?;
let mut bytes = vec![0u8; len];
file.read_exact(&mut bytes)
.await
.map_err(|e| RqError::Source(format!("{}: {e}", path.display())))?;
Ok(bytes)
}
struct ControlSourcePreparedTransfer {
entries: Vec<RqSourceEntry>,
object_sizes: Vec<u64>,
manifest: TransferManifest,
precomputed_logical_digests: Vec<EntryDigest>,
_pack_tempdir: Option<tempfile::TempDir>,
}
struct ControlSourceStreamDigestReport {
bytes_streamed: u64,
entry_digests: Vec<ObjectCompleteEntryDigest>,
logical_digests: Vec<ObjectCompleteLogicalDigest>,
merkle_root_hex: String,
}
async fn prepare_control_source_transfer(
root_name: String,
is_directory: bool,
raw_entries: Vec<RqSourceEntry>,
config: &RqConfig,
expected_total_bytes: u64,
) -> Result<ControlSourcePreparedTransfer, RqError> {
let (packed_entries, precomputed_logical_digests, pack_tempdir) =
pack_small_files_with_deferred_singleton_digests(raw_entries, config, true).await?;
let entries = split_large_entries_with_digest_mode(
packed_entries,
&precomputed_logical_digests,
config,
ManifestDigestMode::SourceStreamTrailer,
)
.await?;
let object_sizes = source_entry_sizes(&entries).await?;
let total_bytes = object_sizes
.iter()
.try_fold(0u64, |acc, size| acc.checked_add(*size))
.ok_or(RqError::TooLarge {
size: u64::MAX,
max: config.max_transfer_bytes,
})?;
if total_bytes != expected_total_bytes {
return Err(RqError::Source(format!(
"source entry sizes changed while preparing control stream: expected {expected_total_bytes}, got {total_bytes}"
)));
}
if total_bytes > config.max_transfer_bytes {
return Err(RqError::TooLarge {
size: total_bytes,
max: config.max_transfer_bytes,
});
}
let manifest_entries: Vec<ManifestEntry> = entries
.iter()
.zip(object_sizes.iter())
.enumerate()
.map(|(i, (entry, size))| ManifestEntry {
index: u32::try_from(i).unwrap_or(u32::MAX),
rel_path: entry.rel_path.clone(),
size: *size,
sha256_hex: sha256_hex_placeholder(),
members: entry.members.clone(),
fragment: entry.fragment.clone(),
})
.collect();
let transfer_id =
transfer_id_hex_from_structure(&root_name, is_directory, total_bytes, &manifest_entries);
let manifest = TransferManifest {
transfer_id,
root_name,
is_directory,
total_bytes,
merkle_root_hex: sha256_hex_placeholder(),
entries: manifest_entries,
};
Ok(ControlSourcePreparedTransfer {
entries,
object_sizes,
manifest,
precomputed_logical_digests,
_pack_tempdir: pack_tempdir,
})
}
async fn stream_control_source_entries<S>(
cx: &Cx,
control: &mut FrameTransport<S>,
encoders: &[EntryEncoder],
manifest: &TransferManifest,
precomputed_logical_digests: &[EntryDigest],
transfer_id: &str,
symbol_auth: Option<&SecurityContext>,
) -> Result<ControlSourceStreamDigestReport, RqError>
where
S: AsyncReadExt + AsyncWriteExt + Unpin,
{
let mut buf = vec![0u8; control_source_data_chunk_bytes(symbol_auth.is_some())];
let mut bytes_streamed = 0u64;
let mut chunks = 0u64;
let mut pending_flush_bytes = 0usize;
let mut flushes = 0u64;
let mut entry_digests = Vec::with_capacity(encoders.len());
let mut logical_digests: Vec<EntryDigest> = precomputed_logical_digests.to_vec();
let mut logical_fragment_hashes: BTreeMap<
String,
crate::net::atp::transport_common::StagedEntryReceive,
> = BTreeMap::new();
for enc in encoders {
cx.checkpoint().map_err(|_| RqError::Cancelled)?;
let manifest_entry = manifest
.entries
.iter()
.find(|entry| entry.index == enc.index)
.ok_or_else(|| {
RqError::Coding(format!(
"control source encoder {} missing manifest entry",
enc.index
))
})?;
let mut file = crate::fs::File::open(&enc.abs_path)
.await
.map_err(|e| RqError::Source(format!("{}: {e}", enc.abs_path.display())))?;
let source_offset = u64::try_from(enc.source_offset).map_err(|_| {
RqError::Source(format!(
"{}: source offset does not fit u64: {}",
enc.abs_path.display(),
enc.source_offset
))
})?;
file.seek(std::io::SeekFrom::Start(source_offset))
.await
.map_err(|e| RqError::Source(format!("{}: {e}", enc.abs_path.display())))?;
let mut remaining = u64::try_from(enc.size).map_err(|_| RqError::TooLarge {
size: u64::MAX,
max: u64::try_from(usize::MAX).unwrap_or(u64::MAX),
})?;
let mut offset = 0u64;
let mut entry_hash =
crate::net::atp::transport_common::StagedEntryReceive::new(enc.abs_path.clone());
while remaining > 0 {
cx.checkpoint().map_err(|_| RqError::Cancelled)?;
let want = usize::try_from(remaining.min(buf.len() as u64)).unwrap_or(buf.len());
let n = file
.read(&mut buf[..want])
.await
.map_err(|e| RqError::Source(format!("{}: {e}", enc.abs_path.display())))?;
if n == 0 {
return Err(RqError::Source(format!(
"{}: short read while streaming control source entry {}",
enc.abs_path.display(),
enc.index
)));
}
entry_hash.update_with_chunk(&buf[..n]);
if let Some(fragment) = manifest_entry.fragment.as_ref() {
let logical_hash = logical_fragment_hashes
.entry(fragment.rel_path.clone())
.or_insert_with(|| {
crate::net::atp::transport_common::StagedEntryReceive::new(PathBuf::from(
&fragment.rel_path,
))
});
logical_hash.update_with_chunk(&buf[..n]);
}
let written = control
.send_control_source_data_unflushed(
transfer_id,
enc.index,
offset,
&buf[..n],
symbol_auth,
)
.await?;
pending_flush_bytes = pending_flush_bytes.saturating_add(written);
if pending_flush_bytes >= RQ_CONTROL_SOURCE_FLUSH_BYTES {
control.flush().await?;
pending_flush_bytes = 0;
flushes = flushes.saturating_add(1);
}
let n_u64 = u64::try_from(n).unwrap_or(u64::MAX);
offset = offset.saturating_add(n_u64);
remaining -= n_u64;
bytes_streamed = bytes_streamed.saturating_add(n_u64);
chunks = chunks.saturating_add(1);
}
let (entry_digest, _path, _created) = entry_hash.finalize(manifest_entry.rel_path.clone());
entry_digests.push(ObjectCompleteEntryDigest {
index: enc.index,
size: entry_digest.size,
sha256_hex: hex_encode(&entry_digest.content_sha256),
});
if manifest_entry.fragment.is_none() && manifest_entry.members.is_empty() {
logical_digests.push(entry_digest);
}
}
if pending_flush_bytes > 0 {
control.flush().await?;
flushes = flushes.saturating_add(1);
}
for (rel_path, logical_hash) in logical_fragment_hashes {
let (digest, _path, _created) = logical_hash.finalize(rel_path);
logical_digests.push(digest);
}
let merkle_root_hex = flat_merkle_root_from_digests(&logical_digests);
let logical_digests = logical_digests
.into_iter()
.map(|digest| ObjectCompleteLogicalDigest {
rel_path: digest.rel_path,
size: digest.size,
sha256_hex: hex_encode(&digest.content_sha256),
})
.collect();
rqtrace!(
"sender: control_source_stream sent chunks={} bytes={} flushes={} flush_threshold_bytes={}",
chunks,
bytes_streamed,
flushes,
RQ_CONTROL_SOURCE_FLUSH_BYTES
);
Ok(ControlSourceStreamDigestReport {
bytes_streamed,
entry_digests,
logical_digests,
merkle_root_hex,
})
}
#[allow(clippy::too_many_arguments)]
async fn spray_source_requests<S>(
cx: &Cx,
control: &mut FrameTransport<S>,
adaptive: &mut RqAdaptiveSendState,
sockets: &mut [UdpSocket],
rr: &mut usize,
symbols_sent: &mut u64,
dropper: &mut u32,
tag: u64,
encoders: &[EntryEncoder],
requests: &[SourceSymbolRequest],
config: &RqConfig,
pacer: &mut RqSprayPacer,
symbol_auth: Option<&SecurityContext>,
udp_send_acceleration: &mut UdpSendAccelerationReport,
) -> Result<(), RqError>
where
S: AsyncReadExt + AsyncWriteExt + Unpin,
{
let mut send_batch = RqPendingSendBatch::new(sockets.len());
for request in requests {
let enc = encoders
.iter()
.find(|enc| enc.index == request.entry)
.ok_or_else(|| {
RqError::Coding(format!(
"receiver requested source symbol for unknown entry {}",
request.entry
))
})?;
let sym = source_symbol_for_request(enc, *request, config).await?;
queue_symbol_datagram(
cx,
control,
adaptive,
sockets,
rr,
symbols_sent,
dropper,
tag,
enc.index,
&sym,
config,
pacer,
symbol_auth,
&mut send_batch,
udp_send_acceleration,
)
.await?;
}
let report = send_batch.flush(sockets, symbols_sent).await?;
udp_send_acceleration.observe_flush_report(report);
service_rq_spray_control(cx, control, adaptive).await?;
rqtrace!(
"sender: retransmitted {} requested source symbols",
requests.len()
);
Ok(())
}
#[allow(clippy::too_many_arguments)]
async fn send_symbol_datagrams<S>(
cx: &Cx,
control: &mut FrameTransport<S>,
adaptive: &mut RqAdaptiveSendState,
sockets: &mut [UdpSocket],
rr: &mut usize,
symbols_sent: &mut u64,
dropper: &mut u32,
tag: u64,
entry: u32,
symbols: &[Symbol],
config: &RqConfig,
pacer: &mut RqSprayPacer,
symbol_auth: Option<&SecurityContext>,
udp_send_acceleration: &mut UdpSendAccelerationReport,
) -> Result<(), RqError>
where
S: AsyncReadExt + AsyncWriteExt + Unpin,
{
let mut send_batch = RqPendingSendBatch::new(sockets.len());
for sym in symbols {
queue_symbol_datagram(
cx,
control,
adaptive,
sockets,
rr,
symbols_sent,
dropper,
tag,
entry,
sym,
config,
pacer,
symbol_auth,
&mut send_batch,
udp_send_acceleration,
)
.await?;
}
let report = send_batch.flush(sockets, symbols_sent).await?;
udp_send_acceleration.observe_flush_report(report);
service_rq_spray_control(cx, control, adaptive).await
}
#[allow(clippy::too_many_arguments)]
async fn queue_symbol_datagram<S>(
cx: &Cx,
control: &mut FrameTransport<S>,
adaptive: &mut RqAdaptiveSendState,
sockets: &mut [UdpSocket],
rr: &mut usize,
symbols_sent: &mut u64,
dropper: &mut u32,
tag: u64,
entry: u32,
sym: &Symbol,
config: &RqConfig,
pacer: &mut RqSprayPacer,
symbol_auth: Option<&SecurityContext>,
send_batch: &mut RqPendingSendBatch,
udp_send_acceleration: &mut UdpSendAccelerationReport,
) -> Result<(), RqError>
where
S: AsyncReadExt + AsyncWriteExt + Unpin,
{
cx.checkpoint().map_err(|_| RqError::Cancelled)?;
if config.debug_drop_one_in > 0 {
*dropper = dropper.wrapping_add(1);
if *dropper % config.debug_drop_one_in == 0 {
return Ok(());
}
}
pacer.before_send(cx).await?;
let auth = symbol_auth.map(|ctx| ctx.sign_symbol(sym));
let dgram =
encode_symbol_datagram(tag, entry, sym, auth.as_ref().map(AuthenticatedSymbol::tag));
let fanout = send_batch.fanout();
let socket_index = *rr % fanout;
*rr = rr.wrapping_add(1);
send_batch.push(socket_index, dgram);
pacer.observe_datagram_sent();
if send_batch.should_flush() {
let report = send_batch.flush(sockets, symbols_sent).await?;
udp_send_acceleration.observe_flush_report(report);
service_rq_spray_control(cx, control, adaptive).await?;
}
Ok(())
}
async fn service_rq_spray_control<S>(
cx: &Cx,
control: &mut FrameTransport<S>,
adaptive: &mut RqAdaptiveSendState,
) -> Result<(), RqError>
where
S: AsyncReadExt + AsyncWriteExt + Unpin,
{
cx.checkpoint().map_err(|_| RqError::Cancelled)?;
while let Some(frame) = control.try_recv_ready().await? {
match frame.frame_type() {
FrameType::KeepAlive => adaptive.mark_control_peer_activity(),
FrameType::Close => {
return Err(RqError::Io(std::io::Error::new(
std::io::ErrorKind::UnexpectedEof,
"peer closed control during RQ spray",
)));
}
// The receiver converged (or wants more) and sent a terminal/feedback frame while we
// are still spraying — a fast-transfer race. Do NOT error: stash the frame so the
// post-spray feedback loop's `recv()` handles it (Proof -> finalize, ObjectRequest ->
// next round). Stop draining now; remaining sprayed symbols are harmless (the receiver
// ignores extras and the sha256+merkle commit gate still verifies). Fixes zz35zq.
_ => {
control.stashed = Some(frame);
break;
}
}
}
if adaptive.next_control_keepalive_due() {
let frame =
Frame::empty(FrameType::KeepAlive).map_err(|err| RqError::Frame(err.to_string()))?;
control.send(&frame).await?;
}
if adaptive.control_liveness_expired() {
return Err(RqError::Io(std::io::Error::new(
std::io::ErrorKind::TimedOut,
format!(
"peer liveness expired during RQ spray after {} missed beacon probes",
adaptive.missed_control_probes()
),
)));
}
Ok(())
}
async fn source_symbol_for_request(
enc: &EntryEncoder,
request: SourceSymbolRequest,
config: &RqConfig,
) -> Result<Symbol, RqError> {
if request.entry != enc.index {
return Err(RqError::Coding(format!(
"source request entry mismatch: request={}, encoder={}",
request.entry, enc.index
)));
}
let symbol_size = usize::from(config.symbol_size.max(1));
let block_start = usize::from(request.sbn)
.checked_mul(config.max_block_size)
.ok_or_else(|| RqError::Coding("source request block offset overflow".to_string()))?;
if block_start >= enc.size {
return Err(RqError::Coding(format!(
"source request block {} outside entry {} ({} bytes)",
request.sbn, enc.index, enc.size
)));
}
let block_len = config.max_block_size.min(enc.size - block_start);
let block_k = block_len.div_ceil(symbol_size).max(1);
let esi = usize::try_from(request.esi)
.map_err(|_| RqError::Coding("source request ESI does not fit usize".to_string()))?;
if esi >= block_k {
return Err(RqError::Coding(format!(
"source request esi {} outside entry {} block {} K={}",
request.esi, enc.index, request.sbn, block_k
)));
}
let start = block_start + esi * symbol_size;
let end = (start + symbol_size).min(block_start + block_len);
let mut buffer = vec![0u8; symbol_size];
if start < end {
let read_start = enc
.source_offset
.checked_add(start)
.ok_or_else(|| RqError::Coding("source request range offset overflow".to_string()))?;
let bytes = read_source_range(&enc.abs_path, read_start, end - start).await?;
buffer[..bytes.len()].copy_from_slice(&bytes);
}
Ok(Symbol::new(
SymbolId::new(enc.object_id, request.sbn, request.esi),
buffer,
SymbolKind::Source,
))
}
// ─── Public API: receive ─────────────────────────────────────────────────────
/// Accept exactly one transfer (one control connection) on `control_listener`,
/// receiving symbols on a freshly-bound UDP socket, write to `dest_dir`, verify,
/// and return a report.
pub async fn receive_once(
cx: &Cx,
control_listener: &TcpListener,
udp_bind_ip: &str,
dest_dir: &Path,
config: RqConfig,
peer_id: &str,
) -> Result<ReceiveReport, RqError> {
let (stream, peer) = match crate::time::timeout(
cx.now(),
config.accept_timeout,
control_listener.accept(),
)
.await
{
Ok(Ok(accepted)) => accepted,
Ok(Err(err)) => return Err(RqError::Io(err)),
Err(_elapsed) => {
return Err(RqError::Io(std::io::Error::new(
std::io::ErrorKind::TimedOut,
format!("accept timed out after {:?}", config.accept_timeout),
)));
}
};
receive_connection(cx, stream, peer, udp_bind_ip, dest_dir, config, peer_id).await
}
/// Drive a single accepted control connection through the receive protocol.
pub async fn receive_connection(
cx: &Cx,
stream: TcpStream,
peer: SocketAddr,
udp_bind_ip: &str,
dest_dir: &Path,
config: RqConfig,
peer_id: &str,
) -> Result<ReceiveReport, RqError> {
let symbol_auth = config.symbol_auth_context()?;
let symbol_auth_enabled = symbol_auth.is_some();
let should_tune_for_bulk_source = near_clean_control_source_stream_round0(&config);
if should_tune_for_bulk_source {
tune_control_stream_for_bulk_source(&stream);
}
let mut control = FrameTransport::new(stream);
// Handshake.
let hello_frame = control.recv().await?;
if hello_frame.frame_type() != FrameType::Handshake {
return Err(RqError::Unexpected {
got: hello_frame.frame_type(),
expected: "Handshake",
});
}
let hello: Hello = parse_json(&hello_frame)?;
let accepted = hello.protocol == ATP_RQ_PROTOCOL && hello.symbol_auth == symbol_auth_enabled;
let control_source_stream = accepted
&& hello.prefer_control_source_stream
&& control_source_stream_eligible(hello.total_bytes, &config);
let (udp, udp_ports, udp_port) = if control_source_stream {
rqtrace!(
"receiver: control_source_stream accepted total_bytes={}",
hello.total_bytes
);
(None, Vec::new(), 0)
} else {
// Bind the UDP data sockets before acking so the sender can spray immediately.
// Build an owned `SocketAddr` (Copy + 'static) so it satisfies
// `UdpSocket::bind`'s `'static` address bound and handles IPv6 correctly.
let bind_ip: std::net::IpAddr = udp_bind_ip
.parse()
.map_err(|e| RqError::Source(format!("invalid UDP bind ip '{udp_bind_ip}': {e}")))?;
// Size the receive buffer to ABSORB the sender's symbol burst: the sender now encodes
// blocks in parallel (F3) and can spray them faster than the CPU-bound decode drains, so
// we set the buffer to the transfer size plus headroom, clamped to a generous cap (the
// kernel further caps at net.core.rmem_max). For a transfer that fits, the whole burst
// lands in the buffer with no kernel drops and the decoder drains at its own pace. The
// control-source fast lane skips this allocation entirely.
let recv_buf_bytes = if hello.total_bytes == 0 {
16 * 1024 * 1024
} else {
usize::try_from(hello.total_bytes.saturating_add(32 * 1024 * 1024))
.unwrap_or(usize::MAX)
.clamp(16 * 1024 * 1024, 120 * 1024 * 1024)
};
let udp =
RqReceiverUdpFanout::bind(bind_ip, config.udp_fanout.max(1), recv_buf_bytes).await?;
let udp_ports = udp.local_ports()?;
let udp_port = udp_ports.first().copied().ok_or_else(|| {
RqError::Io(std::io::Error::new(
std::io::ErrorKind::NotConnected,
"RQ receiver UDP fanout has no bound sockets",
))
})?;
rqtrace!(
"receiver: udp fanout sockets={} ports={:?}",
udp.len(),
udp_ports
);
(Some(udp), udp_ports, udp_port)
};
control
.send(&json_frame(
FrameType::HandshakeAck,
&HelloAck {
accepted,
peer_id: peer_id.to_string(),
udp_port,
udp_ports,
control_source_stream,
reason: if accepted {
None
} else if hello.protocol != ATP_RQ_PROTOCOL {
Some(format!(
"unsupported protocol {} (this peer speaks {ATP_RQ_PROTOCOL})",
hello.protocol
))
} else if hello.symbol_auth != symbol_auth_enabled {
Some(format!(
"symbol authentication mismatch: sender={}, receiver={symbol_auth_enabled}",
hello.symbol_auth
))
} else {
Some("handshake rejected".to_string())
},
},
)?)
.await?;
if !accepted {
return Err(RqError::HandshakeRejected(
if hello.protocol != ATP_RQ_PROTOCOL {
format!("unsupported protocol {}", hello.protocol)
} else if hello.symbol_auth != symbol_auth_enabled {
format!(
"symbol authentication mismatch: sender={}, receiver={symbol_auth_enabled}",
hello.symbol_auth
)
} else {
"handshake rejected".to_string()
},
));
}
// Manifest.
let manifest_frame = control.recv().await?;
if manifest_frame.frame_type() != FrameType::ObjectManifest {
return Err(RqError::Unexpected {
got: manifest_frame.frame_type(),
expected: "ObjectManifest",
});
}
let manifest = parse_and_validate_manifest_frame(&manifest_frame, &config)?;
if hello.total_bytes != 0 && hello.total_bytes != manifest.total_bytes {
return Err(RqError::Frame(format!(
"handshake total_bytes {} does not match manifest total_bytes {}",
hello.total_bytes, manifest.total_bytes
)));
}
let symbol_size = hello.symbol_size;
let receiver_max_block_size = usize::try_from(hello.max_block_size).map_err(|_| {
RqError::Frame(format!(
"peer max_block_size {} does not fit usize",
hello.max_block_size
))
})?;
let staging_dir = create_receive_staging_dir(dest_dir, &manifest.transfer_id).await?;
let _staging_guard = RqStagingDirGuard::new(staging_dir.clone());
let single_file_fragment_staging = single_file_fragment_staging_path(&manifest, &staging_dir);
let source_streaming = config.repair_overhead <= 1.0 && config.source_retransmit_rounds > 0;
// Per-entry decoders.
let mut decoders: Vec<EntryDecoder> = manifest
.entries
.iter()
.map(|e| {
let object_id = entry_object_id(&manifest.transfer_id, e.index);
let (
staging_path,
staging_write_offset,
staging_file_len,
staging_shared,
) = receive_staging_layout_for_entry(
e,
&staging_dir,
single_file_fragment_staging.as_deref(),
);
let (pipeline, entry_source_streaming, source_blocks) = if control_source_stream {
(None, false, Vec::new())
} else {
let dconfig = DecodingConfig {
symbol_size,
max_block_size: receiver_max_block_size,
repair_overhead: config.repair_overhead,
min_overhead: 0,
// RQ repair rows are round-critical: dropping an undecoded
// block's repair symbols makes the sender re-spray another
// round. Keep them until block completion; mark_block_complete
// clears the block immediately after decode.
max_buffered_symbols: RQ_REPAIR_RECEIVE_SYMBOL_CAP_PER_BLOCK,
block_timeout: std::time::Duration::from_secs(0),
verify_auth: symbol_auth_enabled,
};
let mut pipeline = if let Some(context) = &symbol_auth {
DecodingPipeline::with_auth(dconfig, context.clone())
} else {
DecodingPipeline::new(dconfig)
};
let params =
object_params_for(object_id, e.size, symbol_size, hello.max_block_size);
// set_object_params failure is a metadata bug, surfaced on first feed.
if let Err(err) = pipeline.set_object_params(params) {
rqtrace!(
"receiver: entry {} set_object_params FAILED: {err:?} (size={}, blocks={}, k={})",
e.index,
e.size,
params.source_blocks,
params.symbols_per_block
);
}
let source_blocks =
source_block_progress_for(e.size, receiver_max_block_size, symbol_size);
let entry_source_streaming = source_streaming && source_blocks.is_some();
(
Some(pipeline),
entry_source_streaming,
source_blocks.unwrap_or_default(),
)
};
EntryDecoder {
index: e.index,
object_id,
size: e.size,
pipeline,
complete: e.size == 0,
staging_path: staging_path.clone(),
staging_write_offset,
staging_file_len,
staging_shared,
staging_created: false,
staging_file: None,
staging_cursor: None,
staging_unflushed_bytes: 0,
cache_staging_file: should_cache_entry_staging_file(
e.size,
manifest.entries.len(),
e.members.len(),
),
bytes_written: 0,
max_block_size: receiver_max_block_size,
source_streaming: entry_source_streaming,
source_blocks,
pending_decodes: Vec::new(),
inc: control_source_stream.then(|| {
crate::net::atp::transport_common::StagedEntryReceive::new(staging_path)
}),
inc_digest: None,
source_write_buffer: if control_source_stream {
Vec::new()
} else {
Vec::with_capacity(RQ_SOURCE_STAGE_BUFFER_BYTES)
},
source_write_buffer_offset: None,
}
})
.collect();
if control_source_stream {
return receive_control_source_stream(
cx,
&mut control,
&manifest,
symbol_auth.as_ref(),
&mut decoders,
dest_dir,
peer,
)
.await;
}
let mut udp = udp.expect("UDP receiver is bound for non-control-source transfers");
let tag = transfer_tag(&manifest.transfer_id);
let mut symbols_accepted: u64 = 0;
let mut round_stats = RqDatagramRoundStats::default();
let mut feedback_rounds: u32 = 0;
let trace_receiver_intake = std::env::var_os("ATP_RQ_TRACE").is_some();
let datagram_header_len = if symbol_auth_enabled {
AUTH_DGRAM_HEADER
} else {
DGRAM_HEADER
};
let mut rbuf = vec![0u8; usize::from(symbol_size) + datagram_header_len + 64];
let mut round_wall_start = trace_receiver_intake.then(Instant::now);
// Drive: alternate between draining UDP symbols and responding to the
// sender's ObjectComplete on the control channel. We pump UDP between
// control messages by racing a short-bounded recv against control readiness.
loop {
cx.checkpoint().map_err(|_| RqError::Cancelled)?;
// First, drain any control message that is ready (ObjectComplete ends a
// spray round). We do a blocking control.recv() because the sender only
// sends ObjectComplete after finishing a spray round, and we have been
// consuming UDP concurrently via the pump below.
//
// To keep v1 correct on the current runtime without a select primitive,
// we structure it as: pump UDP until the control frame arrives.
let frame = pump_until_control(
cx,
&mut control,
&mut udp,
tag,
symbol_auth_enabled,
symbol_auth.as_ref(),
&mut rbuf,
&mut decoders,
symbol_size,
&mut symbols_accepted,
&mut round_stats,
trace_receiver_intake,
)
.await?;
rqtrace!(
"receiver: pump returned {:?}, symbols_accepted={symbols_accepted}",
frame.frame_type()
);
match frame.frame_type() {
FrameType::ObjectComplete => {
let round_complete = parse_round_complete(&frame)?;
let completion_digests =
CompletionDigestIndex::from_round_complete(&round_complete, &manifest, false)?;
let drained = drain_round_tail(
cx,
&mut udp,
tag,
symbol_auth_enabled,
symbol_auth.as_ref(),
&mut rbuf,
config.round_tail_drain,
&mut decoders,
symbol_size,
&mut symbols_accepted,
&mut round_stats,
trace_receiver_intake,
)
.await?;
if drained > 0 {
rqtrace!("receiver: tail-drained {drained} datagrams after ObjectComplete");
}
let seed_stats = flush_and_seed_source_streaming_round_boundary(
cx,
&mut decoders,
symbol_size,
symbol_auth.as_ref(),
)
.await?;
round_stats.record_decode_stats(seed_stats.decode_stats);
if seed_stats.seeded > 0 {
rqtrace!(
"receiver: seeded {} source-streaming block(s) at round boundary",
seed_stats.seeded
);
}
let decode_width_budget = rq_decode_width_budget_for_cx(cx, &decoders, symbol_size);
let pending_decode_jobs_before_join = rq_pending_decode_jobs(&decoders);
let completed_decode_stats =
join_all_pending_decodes(cx, &mut decoders, decode_width_budget).await?;
round_stats.record_decode_stats(completed_decode_stats);
let pending_decode_jobs_after_join = rq_pending_decode_jobs(&decoders);
if completed_decode_stats.attempts > 0 {
rqtrace!(
"receiver: finalized {} pending decode job(s) after ObjectComplete (decode_repair_attempts={} decode_source_complete_attempts={} completed_blocks={} stale_requeues={} decode_micros={} decode_join_wait_micros={} decode_apply_micros={} decode_persist_micros={} decode_queued_jobs={} decode_inline_jobs={} decode_spawn_denials={} decode_entry_cap_saturations={} decode_transfer_cap_saturations={} decode_pending_peak={})",
completed_decode_stats.attempts,
completed_decode_stats.repair_attempts,
completed_decode_stats.source_complete_attempts,
completed_decode_stats.completed_blocks,
completed_decode_stats.stale_requeues,
completed_decode_stats.decode_micros,
completed_decode_stats.join_wait_micros,
completed_decode_stats.apply_micros,
completed_decode_stats.persist_micros,
completed_decode_stats.queued_jobs,
completed_decode_stats.inline_jobs,
completed_decode_stats.spawn_denials,
completed_decode_stats.entry_cap_saturations,
completed_decode_stats.transfer_cap_saturations,
completed_decode_stats.pending_peak
);
}
flush_cached_entry_staging_files(&mut decoders).await?;
let pending: Vec<u32> = decoders
.iter()
.filter(|d| !d.complete)
.map(|d| d.index)
.collect();
let round_loss_fraction = receiver_round_loss_fraction(
round_stats.observed,
round_complete.round_symbols_sent,
);
let decode_budget =
rq_decode_width_budget_snapshot_for_cx(cx, &decoders, symbol_size);
let round_wall_micros = elapsed_micros_since(round_wall_start);
rqtrace!(
"receiver: ObjectComplete; {} of {} entries still pending round_symbols_sent={} round_symbols_observed={} round_symbols_accepted={} round_source_observed={} round_source_accepted={} round_repair_observed={} round_repair_accepted={} round_loss_fraction={:.4} intake_payload_bytes={} round_wall_micros={} round_wall_symbols_per_s={} round_wall_bytes_per_s={} intake_micros={} intake_symbols_per_s={} intake_bytes_per_s={} parse_micros={} feed_micros={} source_auth_micros={} source_persist_micros={} pipeline_feed_micros={} block_persist_micros={} decode_dispatch_micros={} source_seed_micros={} feed_other_micros={} recv_micros={} drain_micros={} decode_attempts={} decode_repair_attempts={} decode_source_complete_attempts={} decode_completed_blocks={} decode_stale_requeues={} decode_micros={} decode_join_wait_micros={} decode_apply_micros={} decode_persist_micros={} decode_queued_jobs={} decode_inline_jobs={} decode_spawn_denials={} decode_entry_cap_saturations={} decode_transfer_cap_saturations={} decode_pending_peak={} pending_decode_jobs_before_join={} pending_decode_jobs_after_join={} decode_width_budget={} decode_core_limit={} decode_memory_limit={} decode_job_memory_bytes={} decode_max_block_bytes={}",
pending.len(),
decoders.len(),
round_complete.round_symbols_sent,
round_stats.observed,
round_stats.accepted,
round_stats.source_observed,
round_stats.source_accepted,
round_stats.repair_observed,
round_stats.repair_accepted,
round_loss_fraction.unwrap_or(0.0),
round_stats.payload_bytes,
round_wall_micros,
rate_per_second(round_stats.observed, round_wall_micros),
rate_per_second(round_stats.payload_bytes, round_wall_micros),
round_stats.intake_micros(),
round_stats.intake_symbols_per_s(),
round_stats.intake_bytes_per_s(),
round_stats.parse_micros,
round_stats.feed_micros,
round_stats.source_auth_micros,
round_stats.source_persist_micros,
round_stats.pipeline_feed_micros,
round_stats.block_persist_micros,
round_stats.decode_dispatch_micros,
round_stats.source_seed_micros,
round_stats.feed_other_micros,
round_stats.recv_micros,
round_stats.drain_micros,
round_stats.decode_stats.attempts,
round_stats.decode_stats.repair_attempts,
round_stats.decode_stats.source_complete_attempts,
round_stats.decode_stats.completed_blocks,
round_stats.decode_stats.stale_requeues,
round_stats.decode_stats.decode_micros,
round_stats.decode_stats.join_wait_micros,
round_stats.decode_stats.apply_micros,
round_stats.decode_stats.persist_micros,
round_stats.decode_stats.queued_jobs,
round_stats.decode_stats.inline_jobs,
round_stats.decode_stats.spawn_denials,
round_stats.decode_stats.entry_cap_saturations,
round_stats.decode_stats.transfer_cap_saturations,
round_stats.decode_stats.pending_peak,
pending_decode_jobs_before_join,
pending_decode_jobs_after_join,
decode_budget.effective,
decode_budget.core_limit,
decode_budget.memory_limit,
decode_budget.job_memory_bytes,
decode_budget.max_block_size
);
trace_receiver_decode_profile(
"ObjectComplete",
feedback_rounds,
round_stats.decode_stats,
decode_budget.effective,
);
if pending.is_empty() {
// Verify + commit + Proof.
let receipt = verify_and_commit(
&manifest,
&mut decoders,
dest_dir,
symbols_accepted,
feedback_rounds,
&std::collections::BTreeMap::new(),
&completion_digests,
)
.await?;
control
.send(&json_frame(FrameType::Proof, &receipt)?)
.await?;
drain_sender_close_after_proof(cx, &mut control, "udp-round").await;
if !receipt.committed {
return Err(RqError::Integrity(
receipt
.reason
.unwrap_or_else(|| "verification failed".to_string()),
));
}
let committed_paths: Vec<PathBuf> =
receipt.committed_paths.iter().map(PathBuf::from).collect();
return Ok(ReceiveReport {
transfer_id: manifest.transfer_id,
bytes_received: receipt.bytes_received,
files: receipt.files,
committed: true,
symbols_accepted,
feedback_rounds,
committed_paths,
peer,
});
}
// Ask for more symbols for the pending entries.
feedback_rounds += 1;
if feedback_rounds > config.max_feedback_rounds {
let receipt = ReceiveReceipt {
committed: false,
bytes_received: 0,
files: u32::try_from(manifest.entries.len()).unwrap_or(u32::MAX),
sha_ok: false,
merkle_ok: false,
symbols_accepted,
feedback_rounds,
reason: Some(format!(
"no convergence after {feedback_rounds} rounds, {} entries pending",
pending.len()
)),
committed_paths: Vec::new(),
};
let _ = control.send(&json_frame(FrameType::Proof, &receipt)?).await;
return Err(RqError::NoConvergence {
rounds: feedback_rounds,
pending: pending.len(),
});
}
let source_symbols = source_retransmit_request_limit(&config, feedback_rounds)
.map_or_else(Vec::new, |limit| collect_source_requests(&decoders, limit));
let progress = source_progress_for_pending(&decoders, &pending);
if trace_receiver_intake {
rqtrace!(
"receiver: NeedMore round={feedback_rounds} pending={} source_requests={} round_symbols_sent={} round_symbols_observed={} round_symbols_accepted={} round_source_observed={} round_source_accepted={} round_repair_observed={} round_repair_accepted={} round_loss_fraction={:.4} symbols_accepted={} intake_payload_bytes={} round_wall_micros={} round_wall_symbols_per_s={} round_wall_bytes_per_s={} intake_micros={} intake_symbols_per_s={} intake_bytes_per_s={} parse_micros={} feed_micros={} source_auth_micros={} source_persist_micros={} pipeline_feed_micros={} block_persist_micros={} decode_dispatch_micros={} source_seed_micros={} feed_other_micros={} recv_micros={} drain_micros={} decode_attempts={} decode_repair_attempts={} decode_source_complete_attempts={} decode_completed_blocks={} decode_stale_requeues={} decode_micros={} decode_join_wait_micros={} decode_apply_micros={} decode_persist_micros={} decode_queued_jobs={} decode_inline_jobs={} decode_spawn_denials={} decode_entry_cap_saturations={} decode_transfer_cap_saturations={} decode_pending_peak={} decode_width_budget={} decode_core_limit={} decode_memory_limit={} decode_job_memory_bytes={} decode_max_block_bytes={} source_received={}/{} pending_decode_jobs={} rank={}/{} rank_deficit={} rank_blocks={}",
pending.len(),
source_symbols.len(),
round_complete.round_symbols_sent,
round_stats.observed,
round_stats.accepted,
round_stats.source_observed,
round_stats.source_accepted,
round_stats.repair_observed,
round_stats.repair_accepted,
round_loss_fraction.unwrap_or(0.0),
symbols_accepted,
round_stats.payload_bytes,
round_wall_micros,
rate_per_second(round_stats.observed, round_wall_micros),
rate_per_second(round_stats.payload_bytes, round_wall_micros),
round_stats.intake_micros(),
round_stats.intake_symbols_per_s(),
round_stats.intake_bytes_per_s(),
round_stats.parse_micros,
round_stats.feed_micros,
round_stats.source_auth_micros,
round_stats.source_persist_micros,
round_stats.pipeline_feed_micros,
round_stats.block_persist_micros,
round_stats.decode_dispatch_micros,
round_stats.source_seed_micros,
round_stats.feed_other_micros,
round_stats.recv_micros,
round_stats.drain_micros,
round_stats.decode_stats.attempts,
round_stats.decode_stats.repair_attempts,
round_stats.decode_stats.source_complete_attempts,
round_stats.decode_stats.completed_blocks,
round_stats.decode_stats.stale_requeues,
round_stats.decode_stats.decode_micros,
round_stats.decode_stats.join_wait_micros,
round_stats.decode_stats.apply_micros,
round_stats.decode_stats.persist_micros,
round_stats.decode_stats.queued_jobs,
round_stats.decode_stats.inline_jobs,
round_stats.decode_stats.spawn_denials,
round_stats.decode_stats.entry_cap_saturations,
round_stats.decode_stats.transfer_cap_saturations,
round_stats.decode_stats.pending_peak,
decode_budget.effective,
decode_budget.core_limit,
decode_budget.memory_limit,
decode_budget.job_memory_bytes,
decode_budget.max_block_size,
progress.source_received,
progress.source_needed,
progress.pending_decode_jobs,
progress.rank,
progress.rank_columns,
progress.rank_deficit,
progress.rank_blocks,
);
trace_receiver_decode_profile(
"NeedMore",
feedback_rounds,
round_stats.decode_stats,
decode_budget.effective,
);
}
control
.send(&json_frame(
FrameType::ObjectRequest,
&NeedMore {
pending,
source_symbols,
round_symbols_observed: Some(round_stats.observed),
round_symbols_accepted: Some(round_stats.accepted),
round_loss_fraction,
pending_rank: Some(usize_to_u64(progress.rank)),
pending_rank_columns: Some(usize_to_u64(progress.rank_columns)),
pending_rank_deficit: Some(usize_to_u64(progress.rank_deficit)),
pending_decode_jobs: Some(usize_to_u64(progress.pending_decode_jobs)),
},
)?)
.await?;
round_stats = RqDatagramRoundStats::default();
round_wall_start = trace_receiver_intake.then(Instant::now);
}
FrameType::KeepAlive => {
control
.send(
&Frame::empty(FrameType::KeepAlive)
.map_err(|e| RqError::Frame(e.to_string()))?,
)
.await?;
}
FrameType::Close => {
return Err(RqError::Io(std::io::Error::new(
std::io::ErrorKind::UnexpectedEof,
"sender closed control before transfer completed",
)));
}
other => {
return Err(RqError::Unexpected {
got: other,
expected: "ObjectComplete | KeepAlive",
});
}
}
}
}
#[allow(clippy::too_many_arguments)]
async fn apply_control_source_data_frame(
frame: &Frame,
transfer_id: &str,
symbol_auth: Option<&SecurityContext>,
decoders: &mut [EntryDecoder],
manifest: &TransferManifest,
logical: &mut BTreeMap<String, crate::net::atp::transport_common::StagedEntryReceive>,
logical_done: &mut BTreeMap<String, (u64, crate::atp::object::ObjectId, [u8; 32])>,
) -> Result<usize, RqError> {
let data = parse_control_source_data_frame(frame, transfer_id, symbol_auth)?;
let pos = decoder_position_for_entry(decoders, data.entry).ok_or_else(|| {
RqError::Frame(format!(
"control source ObjectData for unknown entry {}",
data.entry
))
})?;
let dec = &mut decoders[pos];
let len_u64 = u64::try_from(data.data.len()).unwrap_or(u64::MAX);
let end = data.offset.checked_add(len_u64).ok_or_else(|| {
RqError::Frame(format!(
"control source ObjectData entry {} offset overflow",
data.entry
))
})?;
if data.offset != dec.bytes_written {
return Err(RqError::Frame(format!(
"control source ObjectData entry {} offset {} does not match expected {}",
data.entry, data.offset, dec.bytes_written
)));
}
if end > dec.size {
return Err(RqError::Frame(format!(
"control source ObjectData entry {} overruns declared size {}",
data.entry, dec.size
)));
}
if data.data.is_empty() {
return Ok(0);
}
write_entry_staging_range(dec, data.offset, data.data).await?;
// Fold the chunk into the incremental digest in receive order. The offset
// == bytes_written guard above guarantees strictly in-order contiguous
// bytes, so this equals a post-stream hash of the staged file.
if let Some(inc) = dec.inc.as_mut() {
inc.update_with_chunk(data.data);
}
// L2: for a fragmented logical file, also fold the chunk into a per-logical-
// file running hash. Fragments stream in shard order and each is in-order,
// so the concatenation of arrival-order chunks equals the logical file; this
// lets commit skip the post-stream re-read of every fragment (the remaining
// clean-link tail after the per-fragment inc-hash).
if let Some(frag) = manifest
.entries
.iter()
.find(|e| e.index == data.entry)
.and_then(|e| e.fragment.as_ref())
{
let lh = logical.entry(frag.rel_path.clone()).or_insert_with(|| {
crate::net::atp::transport_common::StagedEntryReceive::new(std::path::PathBuf::from(
&frag.rel_path,
))
});
lh.update_with_chunk(data.data);
if lh.bytes_written == frag.logical_size {
if let Some(done) = logical.remove(&frag.rel_path) {
let (d, _p, _c) = done.finalize(String::new());
logical_done.insert(
frag.rel_path.clone(),
(d.size, d.content_id, d.content_sha256),
);
}
}
}
dec.bytes_written = end;
if dec.bytes_written == dec.size {
dec.complete = true;
dec.pipeline = None;
close_cached_entry_staging_file(dec).await?;
if let Some(inc) = dec.inc.take() {
let (digest, _path, _created) = inc.finalize(String::new());
dec.inc_digest = Some((digest.size, digest.content_id, digest.content_sha256));
}
}
Ok(data.data.len())
}
async fn receive_control_source_stream<S>(
cx: &Cx,
control: &mut FrameTransport<S>,
manifest: &TransferManifest,
symbol_auth: Option<&SecurityContext>,
decoders: &mut [EntryDecoder],
dest_dir: &Path,
peer: SocketAddr,
) -> Result<ReceiveReport, RqError>
where
S: AsyncReadExt + AsyncWriteExt + Unpin,
{
let mut bytes_streamed = 0u64;
let mut chunks = 0u64;
// L2: per-logical-file running hashes for fragmented objects, finalized in
// arrival order so commit can skip the post-stream fragment re-read.
let mut logical: BTreeMap<String, crate::net::atp::transport_common::StagedEntryReceive> =
BTreeMap::new();
let mut logical_done: BTreeMap<String, (u64, crate::atp::object::ObjectId, [u8; 32])> =
BTreeMap::new();
loop {
cx.checkpoint().map_err(|_| RqError::Cancelled)?;
let frame = control.recv().await?;
match frame.frame_type() {
FrameType::ObjectData => {
let n = apply_control_source_data_frame(
&frame,
&manifest.transfer_id,
symbol_auth,
decoders,
manifest,
&mut logical,
&mut logical_done,
)
.await?;
bytes_streamed =
bytes_streamed.saturating_add(u64::try_from(n).unwrap_or(u64::MAX));
chunks = chunks.saturating_add(1);
}
FrameType::ObjectComplete => {
let complete = parse_round_complete(&frame)?;
let completion_digests =
CompletionDigestIndex::from_round_complete(&complete, manifest, true)?;
flush_cached_entry_staging_files(decoders).await?;
let pending: Vec<u32> = decoders
.iter()
.filter(|decoder| !decoder.complete)
.map(|decoder| decoder.index)
.collect();
if !pending.is_empty() {
let receipt = ReceiveReceipt {
committed: false,
bytes_received: bytes_streamed,
files: u32::try_from(manifest.entries.len()).unwrap_or(u32::MAX),
sha_ok: false,
merkle_ok: false,
symbols_accepted: 0,
feedback_rounds: 0,
reason: Some(format!(
"control source stream ended with {} entries pending",
pending.len()
)),
committed_paths: Vec::new(),
};
let _ = control.send(&json_frame(FrameType::Proof, &receipt)?).await;
return Err(RqError::NoConvergence {
rounds: 0,
pending: pending.len(),
});
}
let receipt = verify_and_commit(
manifest,
decoders,
dest_dir,
0,
0,
&logical_done,
&completion_digests,
)
.await?;
control
.send(&json_frame(FrameType::Proof, &receipt)?)
.await?;
drain_sender_close_after_proof(cx, control, "control-source").await;
if !receipt.committed {
return Err(RqError::Integrity(
receipt
.reason
.unwrap_or_else(|| "verification failed".to_string()),
));
}
rqtrace!(
"receiver: control_source_stream committed chunks={} bytes={}",
chunks,
bytes_streamed
);
let committed_paths: Vec<PathBuf> =
receipt.committed_paths.iter().map(PathBuf::from).collect();
return Ok(ReceiveReport {
transfer_id: manifest.transfer_id.clone(),
bytes_received: receipt.bytes_received,
files: receipt.files,
committed: true,
symbols_accepted: 0,
feedback_rounds: 0,
committed_paths,
peer,
});
}
FrameType::KeepAlive => {
control
.send(
&Frame::empty(FrameType::KeepAlive)
.map_err(|e| RqError::Frame(e.to_string()))?,
)
.await?;
}
FrameType::Close => {
return Err(RqError::Io(std::io::Error::new(
std::io::ErrorKind::UnexpectedEof,
"sender closed control before source stream completed",
)));
}
other => {
return Err(RqError::Unexpected {
got: other,
expected: "ObjectData | ObjectComplete | KeepAlive",
});
}
}
}
}
/// Feed one received symbol into an entry's decoding pipeline. Returns true if
/// the symbol was a well-formed candidate the pipeline accepted or considered
/// (used only for the accepted-datagram counter, not correctness).
fn source_block_progress_for(
size: u64,
max_block_size: usize,
symbol_size: u16,
) -> Option<Vec<SourceBlockProgress>> {
if size == 0 {
return Some(Vec::new());
}
let mut blocks = Vec::new();
let mut start = 0u64;
let block_size = u64::try_from(max_block_size.max(1)).unwrap_or(u64::MAX);
let symbol_size = u64::from(symbol_size.max(1));
while start < size {
if blocks.len() >= MAX_SOURCE_BLOCKS {
return None;
}
let remaining = size - start;
let len_u64 = remaining.min(block_size);
let len = usize::try_from(len_u64).ok()?;
let k = usize::try_from(len_u64.div_ceil(symbol_size)).ok()?.max(1);
blocks.push(SourceBlockProgress {
start,
len,
k,
received: vec![false; k],
pipeline_seeded: vec![false; k],
auth_tags: vec![None; k],
received_count: 0,
complete: false,
});
start = start.checked_add(len_u64)?;
}
Some(blocks)
}
fn collect_source_requests(decoders: &[EntryDecoder], limit: usize) -> Vec<SourceSymbolRequest> {
let mut requests = Vec::new();
for decoder in decoders {
if decoder.complete {
continue;
}
if decoder.source_streaming {
for (sbn, block) in decoder.source_blocks.iter().enumerate() {
if block.complete {
continue;
}
for (esi, received) in block.received.iter().enumerate() {
if *received {
continue;
}
if limit != 0 && requests.len() >= limit {
return requests;
}
let Ok(esi) = u32::try_from(esi) else {
break;
};
requests.push(SourceSymbolRequest {
entry: decoder.index,
sbn: u8::try_from(sbn).unwrap_or(u8::MAX),
esi,
});
}
}
continue;
}
let Some(pipeline) = decoder.pipeline.as_ref() else {
continue;
};
let remaining = if limit == 0 {
0
} else {
limit.saturating_sub(requests.len())
};
if limit != 0 && remaining == 0 {
break;
}
requests.extend(pipeline.missing_source_symbols(remaining).into_iter().map(
|MissingSourceSymbol { sbn, esi }| SourceSymbolRequest {
entry: decoder.index,
sbn,
esi,
},
));
if limit != 0 && requests.len() >= limit {
break;
}
}
requests
}
#[derive(Debug, Default, Clone, Copy)]
struct PendingDecodeProgress {
source_received: usize,
source_needed: usize,
pending_decode_jobs: usize,
rank: usize,
rank_columns: usize,
rank_deficit: usize,
rank_blocks: usize,
}
fn source_progress_for_pending(
decoders: &[EntryDecoder],
pending: &[u32],
) -> PendingDecodeProgress {
let mut progress = PendingDecodeProgress::default();
for decoder in decoders
.iter()
.filter(|decoder| pending.contains(&decoder.index))
{
progress.pending_decode_jobs = progress
.pending_decode_jobs
.saturating_add(decoder.pending_decodes.len());
for (sbn, block) in decoder.source_blocks.iter().enumerate() {
progress.source_received = progress
.source_received
.saturating_add(block.received_count);
progress.source_needed = progress.source_needed.saturating_add(block.k);
let Some(sbn) = u8::try_from(sbn).ok() else {
continue;
};
let Some(status) = decoder
.pipeline
.as_ref()
.and_then(|pipeline| pipeline.block_status(sbn))
else {
continue;
};
let Some(rank) = status.rank else {
continue;
};
let deficit = status.rank_deficit.unwrap_or(0);
progress.rank = progress.rank.saturating_add(rank);
progress.rank_columns = progress
.rank_columns
.saturating_add(rank.saturating_add(deficit));
progress.rank_deficit = progress.rank_deficit.saturating_add(deficit);
progress.rank_blocks = progress.rank_blocks.saturating_add(1);
}
}
progress
}
async fn feed_pipeline_auth_symbol_with_cx(
cx: &Cx,
dec: &mut EntryDecoder,
parsed: &ParsedDatagram,
auth: AuthenticatedSymbol,
symbol_size: u16,
symbol_auth: Option<&SecurityContext>,
preverified: bool,
allow_spawn_decode: bool,
transfer_decode_width: usize,
trace_intake: bool,
) -> Result<RqSymbolFeed, RqError> {
let mut feed = RqSymbolFeed::default();
if dec.pipeline.is_none() {
return Ok(feed);
}
let pipeline_start = trace_intake.then(Instant::now);
let result = if preverified {
dec.pipeline
.as_mut()
.expect("checked above")
.feed_preverified_streaming_block_deferred(auth)
} else {
dec.pipeline
.as_mut()
.expect("checked above")
.feed_streaming_block_deferred(auth)
};
feed.pipeline_feed_micros = feed
.pipeline_feed_micros
.saturating_add(elapsed_micros_since(pipeline_start));
let accepted = match result {
Ok(DeferredSymbolAcceptResult::Decode(job)) => {
let dispatch_start = trace_intake.then(Instant::now);
let dispatch = dispatch_decode_job(
cx,
dec,
job,
"received repair/source symbol",
allow_spawn_decode,
transfer_decode_width,
)
.await?;
feed.decode_dispatch_micros = feed
.decode_dispatch_micros
.saturating_add(elapsed_micros_since(dispatch_start));
feed.decode_stats.merge(dispatch.decode_stats);
true
}
Ok(DeferredSymbolAcceptResult::Immediate(SymbolAcceptResult::Accepted {
received,
needed,
})) => {
if received >= needed || received % 64 == 0 {
rqtrace!(
"receiver: entry {} accepted sbn={} esi={} kind={:?} received={} needed={}",
dec.index,
parsed.sbn,
parsed.esi,
parsed.kind,
received,
needed
);
}
true
}
Ok(DeferredSymbolAcceptResult::Immediate(SymbolAcceptResult::DecodingStarted {
block_sbn,
})) => {
rqtrace!(
"receiver: entry {} started decode block {} via esi={} kind={:?}",
dec.index,
block_sbn,
parsed.esi,
parsed.kind
);
true
}
Ok(DeferredSymbolAcceptResult::Immediate(SymbolAcceptResult::BlockComplete {
block_sbn,
data,
})) => {
let persist_start = trace_intake.then(Instant::now);
persist_decoded_block(dec, block_sbn, &data).await?;
feed.block_persist_micros = feed
.block_persist_micros
.saturating_add(elapsed_micros_since(persist_start));
// `persist_decoded_block` may have already completed the entry via the source-block
// tracker (mixed source+FEC, E-9). Otherwise fall back to the pipeline's own view
// (the all-FEC / non-source-streaming path).
if dec.complete
|| dec
.pipeline
.as_ref()
.is_some_and(DecodingPipeline::is_complete)
{
dec.complete = true;
dec.pipeline = None;
}
rqtrace!(
"receiver: entry {} completed block {} via esi={} kind={:?}",
dec.index,
block_sbn,
parsed.esi,
parsed.kind
);
true
}
Ok(DeferredSymbolAcceptResult::Immediate(SymbolAcceptResult::Duplicate)) => {
rqtrace!(
"receiver: entry {} duplicate sbn={} esi={} kind={:?}",
dec.index,
parsed.sbn,
parsed.esi,
parsed.kind
);
false
}
Ok(DeferredSymbolAcceptResult::Immediate(SymbolAcceptResult::Rejected(reason))) => {
rqtrace!(
"receiver: entry {} rejected sbn={} esi={} kind={:?} reason={:?}",
dec.index,
parsed.sbn,
parsed.esi,
parsed.kind,
reason
);
false
}
Err(err) => {
rqtrace!(
"receiver: entry {} feed error sbn={} esi={} kind={:?}: {err}",
dec.index,
parsed.sbn,
parsed.esi,
parsed.kind
);
false
}
};
if accepted && dec.source_streaming && parsed.kind.is_repair() {
let seed_start = trace_intake.then(Instant::now);
feed.decode_stats.merge(
seed_source_streaming_pipeline(
cx,
dec,
parsed.sbn,
symbol_size,
symbol_auth,
allow_spawn_decode,
transfer_decode_width,
)
.await?,
);
feed.source_seed_micros = feed
.source_seed_micros
.saturating_add(elapsed_micros_since(seed_start));
}
feed.accepted = accepted;
Ok(feed)
}
async fn feed_symbol_with_cx(
cx: &Cx,
dec: &mut EntryDecoder,
parsed: &ParsedDatagram,
payload: &[u8],
symbol_size: u16,
symbol_auth: Option<&SecurityContext>,
allow_spawn_decode: bool,
transfer_decode_width: usize,
trace_intake: bool,
) -> Result<RqSymbolFeed, RqError> {
if dec.complete {
return Ok(RqSymbolFeed::default());
}
if payload.len() != usize::from(symbol_size) {
// RaptorQ symbols are fixed-size; ignore malformed/truncated payloads.
// (The final block's short tail is zero-padded by the encoder, so all
// emitted symbols are symbol_size bytes.)
return Ok(RqSymbolFeed::default());
}
let mut feed = RqSymbolFeed::default();
let mut pipeline_auth = None;
if dec.source_streaming && parsed.kind.is_source() {
if let Some(tag) = parsed.auth_tag {
let Some(context) = symbol_auth else {
return Ok(feed);
};
let sym = Symbol::new(
SymbolId::new(dec.object_id, parsed.sbn, parsed.esi),
payload.to_vec(),
parsed.kind,
);
let mut auth = AuthenticatedSymbol::from_parts(sym, tag);
let auth_start = trace_intake.then(Instant::now);
if context.verify_authenticated_symbol(&mut auth).is_err() {
feed.source_auth_micros = feed
.source_auth_micros
.saturating_add(elapsed_micros_since(auth_start));
rqtrace!(
"receiver: entry {} rejected source-streamed sbn={} esi={} auth tag",
dec.index,
parsed.sbn,
parsed.esi
);
return Ok(feed);
}
feed.source_auth_micros = feed
.source_auth_micros
.saturating_add(elapsed_micros_since(auth_start));
if auth.is_verified() {
let persist_start = trace_intake.then(Instant::now);
feed.accepted = persist_source_symbol(dec, parsed, payload, symbol_size).await?;
feed.source_persist_micros = feed
.source_persist_micros
.saturating_add(elapsed_micros_since(persist_start));
return Ok(feed);
}
pipeline_auth = Some(auth);
} else if symbol_auth.is_some() {
return Ok(feed);
} else {
let persist_start = trace_intake.then(Instant::now);
feed.accepted = persist_source_symbol(dec, parsed, payload, symbol_size).await?;
feed.source_persist_micros = feed
.source_persist_micros
.saturating_add(elapsed_micros_since(persist_start));
return Ok(feed);
}
}
let auth = if let Some(auth) = pipeline_auth {
auth
} else {
let sym = Symbol::new(
SymbolId::new(dec.object_id, parsed.sbn, parsed.esi),
payload.to_vec(),
parsed.kind,
);
if let Some(tag) = parsed.auth_tag {
AuthenticatedSymbol::from_parts(sym, tag)
} else {
AuthenticatedSymbol::new_unauthenticated(sym)
}
};
let pipeline_feed = feed_pipeline_auth_symbol_with_cx(
cx,
dec,
parsed,
auth,
symbol_size,
symbol_auth,
false,
allow_spawn_decode,
transfer_decode_width,
trace_intake,
)
.await?;
feed.pipeline_feed_micros = feed
.pipeline_feed_micros
.saturating_add(pipeline_feed.pipeline_feed_micros);
feed.block_persist_micros = feed
.block_persist_micros
.saturating_add(pipeline_feed.block_persist_micros);
feed.decode_dispatch_micros = feed
.decode_dispatch_micros
.saturating_add(pipeline_feed.decode_dispatch_micros);
feed.source_seed_micros = feed
.source_seed_micros
.saturating_add(pipeline_feed.source_seed_micros);
feed.decode_stats.merge(pipeline_feed.decode_stats);
feed.accepted = pipeline_feed.accepted;
Ok(feed)
}
#[cfg(test)]
async fn feed_symbol(
dec: &mut EntryDecoder,
parsed: &ParsedDatagram,
payload: &[u8],
symbol_size: u16,
symbol_auth: Option<&SecurityContext>,
) -> Result<bool, RqError> {
let cx = Cx::for_testing();
let feed = feed_symbol_with_cx(
&cx,
dec,
parsed,
payload,
symbol_size,
symbol_auth,
true,
RQ_MAX_PENDING_DECODE_JOBS_PER_TRANSFER_HARD,
false,
)
.await?;
while !dec.pending_decodes.is_empty() {
let _ =
join_one_pending_decode(&cx, dec, true, RQ_MAX_PENDING_DECODE_JOBS_PER_TRANSFER_HARD)
.await?;
}
Ok(feed.accepted)
}
async fn dispatch_decode_job(
cx: &Cx,
dec: &mut EntryDecoder,
job: BlockDecodeJob,
trigger: &'static str,
allow_spawn_decode: bool,
transfer_decode_width: usize,
) -> Result<DecodeDispatchOutcome, RqError> {
let block_sbn = job.sbn();
let mut decode_stats =
drain_ready_entry_decodes(cx, dec, allow_spawn_decode, transfer_decode_width).await?;
if block_decode_pending(dec, block_sbn) {
rqtrace!(
"receiver: entry {} dropped duplicate decode job for block {} from {trigger}",
dec.index,
block_sbn
);
if let Some(pipeline) = dec.pipeline.as_mut() {
pipeline.restore_decode_job(job);
}
return Ok(DecodeDispatchOutcome::new(
DecodeDispatch::Queued,
decode_stats,
));
}
let entry_decode_width = entry_decode_width_budget(dec, transfer_decode_width);
if entry_decode_width <= 1 {
while !dec.pending_decodes.is_empty() {
let joined = join_one_pending_decode(cx, dec, false, transfer_decode_width).await?;
decode_stats.merge(joined);
if dec.complete || dec.pipeline.is_none() || block_decode_pending(dec, block_sbn) {
return Ok(DecodeDispatchOutcome::new(
DecodeDispatch::NoProgress,
decode_stats,
));
}
}
decode_stats.merge(
finalize_decode_outcome(
cx,
dec,
run_block_decode_job(job),
false,
transfer_decode_width,
)
.await?,
);
decode_stats.record_inline_job();
rqtrace!(
"receiver: entry {} ran decode block {} inline from {trigger} because entry size/block-count is below the parallel decode gate",
dec.index,
block_sbn
);
return Ok(DecodeDispatchOutcome::new(
DecodeDispatch::NoProgress,
decode_stats,
));
}
if !allow_spawn_decode {
decode_stats.record_transfer_cap_saturation();
let joined = join_one_pending_decode(cx, dec, false, transfer_decode_width).await?;
decode_stats.merge(joined);
rqtrace!(
"receiver: entry {} joined {} pending decode job(s) before queueing block {} from {trigger} because transfer decode width is saturated",
dec.index,
joined.attempts,
block_sbn
);
if dec.complete || dec.pipeline.is_none() || block_decode_pending(dec, block_sbn) {
if let Some(pipeline) = dec.pipeline.as_mut() {
pipeline.restore_decode_job(job);
}
return Ok(DecodeDispatchOutcome::new(
DecodeDispatch::NoProgress,
decode_stats,
));
}
}
if !can_spawn_parallel_decode(dec.pending_decodes.len(), entry_decode_width) {
decode_stats.record_entry_cap_saturation();
let joined = join_one_pending_decode(cx, dec, false, transfer_decode_width).await?;
decode_stats.merge(joined);
rqtrace!(
"receiver: entry {} joined {} pending decode job(s) before queueing block {} from {trigger} (entry_cap={entry_decode_width})",
dec.index,
joined.attempts,
block_sbn
);
if dec.complete || dec.pipeline.is_none() || block_decode_pending(dec, block_sbn) {
if let Some(pipeline) = dec.pipeline.as_mut() {
pipeline.restore_decode_job(job);
}
return Ok(DecodeDispatchOutcome::new(
DecodeDispatch::NoProgress,
decode_stats,
));
}
}
let retry_job = job.clone();
match cx.spawn_blocking(move |_child| run_block_decode_job(job)) {
Ok(handle) => {
dec.pending_decodes
.push(PendingDecode { block_sbn, handle });
decode_stats.record_queued_job(dec.pending_decodes.len());
rqtrace!(
"receiver: entry {} queued parallel decode block {} from {trigger}",
dec.index,
block_sbn
);
Ok(DecodeDispatchOutcome::new(
DecodeDispatch::Queued,
decode_stats,
))
}
Err(crate::runtime::state::SpawnError::RuntimeUnavailable) => {
decode_stats.record_spawn_denial();
decode_stats.merge(
finalize_decode_outcome(
cx,
dec,
run_block_decode_job(retry_job),
false,
transfer_decode_width,
)
.await?,
);
decode_stats.record_inline_job();
rqtrace!(
"receiver: entry {} ran decode block {} inline from {trigger} because no runtime spawn gateway is available",
dec.index,
block_sbn
);
Ok(DecodeDispatchOutcome::new(
DecodeDispatch::NoProgress,
decode_stats,
))
}
Err(err) => {
decode_stats.record_spawn_denial();
let joined = join_one_pending_decode(cx, dec, false, transfer_decode_width).await?;
decode_stats.merge(joined);
rqtrace!(
"receiver: entry {} joined {} pending decode job(s) after spawn denial for block {} from {trigger}: {err:?}",
dec.index,
joined.attempts,
block_sbn
);
if dec.complete || dec.pipeline.is_none() || block_decode_pending(dec, block_sbn) {
if let Some(pipeline) = dec.pipeline.as_mut() {
pipeline.restore_decode_job(retry_job);
}
return Ok(DecodeDispatchOutcome::new(
DecodeDispatch::NoProgress,
decode_stats,
));
}
let restore_job = retry_job.clone();
match cx.spawn_blocking(move |_child| run_block_decode_job(retry_job)) {
Ok(handle) => {
dec.pending_decodes
.push(PendingDecode { block_sbn, handle });
decode_stats.record_queued_job(dec.pending_decodes.len());
rqtrace!(
"receiver: entry {} queued parallel decode block {} from {trigger} after spawn-denial backpressure",
dec.index,
block_sbn
);
Ok(DecodeDispatchOutcome::new(
DecodeDispatch::Queued,
decode_stats,
))
}
Err(retry_err) => {
decode_stats.record_spawn_denial();
rqtrace!(
"receiver: entry {} deferred decode block {} from {trigger} after repeated spawn denial: {retry_err:?}",
dec.index,
block_sbn
);
if let Some(pipeline) = dec.pipeline.as_mut() {
pipeline.restore_decode_job(restore_job);
}
Ok(DecodeDispatchOutcome::new(
DecodeDispatch::NoProgress,
decode_stats,
))
}
}
}
}
}
async fn seed_source_streaming_pipeline(
cx: &Cx,
dec: &mut EntryDecoder,
target_sbn: u8,
symbol_size: u16,
symbol_auth: Option<&SecurityContext>,
allow_spawn_decode: bool,
transfer_decode_width: usize,
) -> Result<RqDecodeRoundStats, RqError> {
let mut decode_stats = RqDecodeRoundStats::default();
if dec.pipeline.is_none() {
return Ok(decode_stats);
}
if block_decode_pending(dec, target_sbn) {
return Ok(decode_stats);
}
let symbol_size = usize::from(symbol_size);
let target_sbn_index = usize::from(target_sbn);
if !source_streaming_block_ready_to_seed(dec, target_sbn_index) {
return Ok(decode_stats);
}
flush_cached_entry_staging_file(dec).await?;
let Some(mut reader) = crate::fs::File::open(&dec.staging_path).await.ok() else {
return Ok(decode_stats);
};
// Seed only the repair block and only once retained repair equations plus staged source
// symbols can reach K. This keeps lossy transfers from retaining source payloads for many
// partially-repaired blocks while still feeding the same symbols before decode.
if dec.source_blocks[target_sbn_index].complete {
return Ok(decode_stats);
}
let sbn = target_sbn_index;
let k = dec.source_blocks[sbn].k;
let block_start = dec.source_blocks[sbn].start;
let block_len = dec.source_blocks[sbn].len;
let mut source_block = vec![0u8; block_len];
// Seed reads MUST use the same shared-staging mapping the writers use:
// `block_start` is ENTRY-relative, but `staging_path` is the SHARED
// fragment for the whole logical object (large entries are E-12 shards
// with `staging_write_offset` bases). Seeking the raw relative offset
// read shard 0's bytes for every later shard, poisoning seeded source
// symbols → InconsistentEquations when redundancy caught it, silent
// rank-K-exact wrong solves (per-entry SHA mismatch) when it did not.
// Entry 0 (base 0) was immune, which is why only later shards rejected
// (c54to7, MATRIX-207).
let absolute_block_start = entry_staging_absolute_offset(dec, block_start, block_len)?;
reader
.seek(std::io::SeekFrom::Start(absolute_block_start))
.await?;
reader.read_exact(&mut source_block).await?;
for esi in 0..k {
let Some((within_block, take, auth_tag)) =
source_seed_symbol_plan(dec, sbn, esi, symbol_size)?
else {
continue;
};
let mut payload = vec![0u8; symbol_size];
payload[..take].copy_from_slice(&source_block[within_block..within_block + take]);
let sbn_u8 = u8::try_from(sbn).map_err(|_| {
RqError::Coding(format!("entry {} source seed SBN overflow", dec.index))
})?;
let esi_u32 = u32::try_from(esi).map_err(|_| {
RqError::Coding(format!("entry {} source seed ESI overflow", dec.index))
})?;
let symbol = Symbol::new(
SymbolId::new(dec.object_id, sbn_u8, esi_u32),
payload,
SymbolKind::Source,
);
let preverified_source_seed = symbol_auth.is_some();
let auth_symbol = if preverified_source_seed {
let tag = auth_tag.ok_or_else(|| {
RqError::Authentication(format!(
"entry {} source seed missing verified auth tag for sbn={sbn} esi={esi}",
dec.index
))
})?;
// Source-streaming tags are stored only after receiver-boundary
// verification succeeds, so seeding the FEC pipeline must not pay a
// second serial HMAC over the same staged source symbol.
AuthenticatedSymbol::new_verified(symbol, tag)
} else {
AuthenticatedSymbol::new_unauthenticated(symbol)
};
let result = if preverified_source_seed {
dec.pipeline
.as_mut()
.expect("checked above")
.feed_preverified_streaming_block_deferred(auth_symbol)
} else {
dec.pipeline
.as_mut()
.expect("checked above")
.feed_streaming_block_deferred(auth_symbol)
};
if result.is_ok() {
dec.source_blocks[sbn].pipeline_seeded[esi] = true;
}
match result {
Ok(DeferredSymbolAcceptResult::Immediate(SymbolAcceptResult::BlockComplete {
block_sbn,
data,
})) => {
persist_decoded_block(dec, block_sbn, &data).await?;
// `persist_decoded_block` may have completed the entry (mixed source+FEC, E-9)
// and already dropped the pipeline; stop seeding so the next loop iteration does
// not touch a `None` pipeline.
if dec.complete
|| dec
.pipeline
.as_ref()
.is_some_and(DecodingPipeline::is_complete)
{
dec.complete = true;
dec.pipeline = None;
return Ok(decode_stats);
}
}
Ok(DeferredSymbolAcceptResult::Immediate(_)) => {}
Ok(DeferredSymbolAcceptResult::Decode(job)) => {
let dispatch = dispatch_decode_job(
cx,
dec,
job,
"source-streaming repair seed",
allow_spawn_decode,
transfer_decode_width,
)
.await?;
decode_stats.merge(dispatch.decode_stats);
match dispatch.dispatch {
DecodeDispatch::Queued => return Ok(decode_stats),
DecodeDispatch::NoProgress => {}
}
}
Err(err) => {
rqtrace!(
"receiver: entry {} source seed error sbn={} esi={}: {err}",
dec.index,
sbn,
esi
);
}
}
}
Ok(decode_stats)
}
async fn flush_and_seed_source_streaming_round_boundary(
cx: &Cx,
decoders: &mut [EntryDecoder],
symbol_size: u16,
symbol_auth: Option<&SecurityContext>,
) -> Result<SourceStreamingSeedStats, RqError> {
for dec in decoders.iter_mut() {
flush_cached_entry_staging_file(dec).await?;
}
let mut seed_stats = SourceStreamingSeedStats::default();
for decoder_index in 0..decoders.len() {
let block_count = decoders[decoder_index].source_blocks.len();
for sbn in 0..block_count {
if decoders[decoder_index].complete || decoders[decoder_index].pipeline.is_none() {
break;
}
if !source_streaming_block_ready_to_seed(&decoders[decoder_index], sbn) {
continue;
}
let transfer_decode_width = rq_decode_width_budget_for_cx(cx, decoders, symbol_size);
let allow_spawn_decode = rq_pending_decode_jobs(decoders) < transfer_decode_width;
let Ok(block_sbn) = u8::try_from(sbn) else {
break;
};
let decode_stats = seed_source_streaming_pipeline(
cx,
&mut decoders[decoder_index],
block_sbn,
symbol_size,
symbol_auth,
allow_spawn_decode,
transfer_decode_width,
)
.await?;
seed_stats.decode_stats.merge(decode_stats);
seed_stats.seeded = seed_stats.seeded.saturating_add(1);
}
}
Ok(seed_stats)
}
async fn persist_source_symbol(
dec: &mut EntryDecoder,
parsed: &ParsedDatagram,
payload: &[u8],
symbol_size: u16,
) -> Result<bool, RqError> {
let sbn = usize::from(parsed.sbn);
if sbn >= dec.source_blocks.len() {
return Ok(false);
}
let Ok(esi) = usize::try_from(parsed.esi) else {
return Ok(false);
};
let symbol_size = usize::from(symbol_size);
let Some(within_block) = esi.checked_mul(symbol_size) else {
return Err(RqError::Coding(format!(
"entry {} source symbol offset overflow",
dec.index
)));
};
let (offset, take) = {
let block = &dec.source_blocks[sbn];
if block.complete || esi >= block.k || block.received[esi] || within_block >= block.len {
return Ok(false);
}
let take = symbol_size.min(block.len - within_block);
let offset = block
.start
.checked_add(u64::try_from(within_block).unwrap_or(u64::MAX))
.ok_or_else(|| {
RqError::Coding(format!("entry {} source symbol offset overflow", dec.index))
})?;
(offset, take)
};
write_source_staging_range(dec, offset, &payload[..take]).await?;
let completed_now = {
let block = &mut dec.source_blocks[sbn];
if block.received[esi] {
return Ok(false);
}
block.received[esi] = true;
block.auth_tags[esi] = parsed.auth_tag;
block.received_count = block.received_count.saturating_add(1);
if block.received_count == block.k {
block.complete = true;
dec.bytes_written = dec
.bytes_written
.checked_add(u64::try_from(block.len).unwrap_or(u64::MAX))
.ok_or_else(|| {
RqError::Coding(format!("entry {} byte counter overflow", dec.index))
})?;
true
} else {
false
}
};
if completed_now {
rqtrace!(
"receiver: entry {} completed source-streamed block {}",
dec.index,
parsed.sbn
);
}
if source_streaming_entry_complete(dec) {
dec.complete = true;
dec.pipeline = None;
close_cached_entry_staging_file(dec).await?;
}
Ok(true)
}
async fn open_entry_staging_file(dec: &mut EntryDecoder) -> Result<crate::fs::File, RqError> {
if let Some(parent) = dec.staging_path.parent() {
crate::fs::create_dir_all(parent).await?;
}
if dec.staging_created {
return Ok(crate::fs::File::options()
.read(true)
.write(true)
.open(&dec.staging_path)
.await?);
}
let file = crate::fs::File::create_new(&dec.staging_path)
.await
.map_err(|err| {
if err.kind() == std::io::ErrorKind::AlreadyExists {
if dec.staging_shared {
RqError::Io(err)
} else {
RqError::Frame(format!(
"staging file already exists for entry {}",
dec.index
))
}
} else {
RqError::Io(err)
}
});
let file = match file {
Ok(file) => file,
Err(RqError::Io(err))
if dec.staging_shared && err.kind() == std::io::ErrorKind::AlreadyExists =>
{
crate::fs::File::options()
.read(true)
.write(true)
.open(&dec.staging_path)
.await?
}
Err(err) => return Err(err),
};
file.set_len(dec.staging_file_len).await?;
dec.staging_created = true;
Ok(file)
}
async fn close_cached_entry_staging_file(dec: &mut EntryDecoder) -> Result<(), RqError> {
flush_source_write_buffer(dec).await?;
if let Some(mut file) = dec.staging_file.take() {
file.flush().await?;
}
dec.staging_cursor = None;
dec.staging_unflushed_bytes = 0;
Ok(())
}
/// Env-gated (`ATP_RQ_INCONSISTENT_AUDIT`) staged-bytes cross-dump for a
/// block whose decode rejected with InconsistentEquations (c54to7): one line
/// per source ESI with the received/seeded bitmap flags and the staged
/// payload fingerprint (same FNV-1a as the decoder-side symbol dump). A
/// seeded symbol whose decoder-side hash differs from the staged hash for
/// the same ESI pins seed-path poisoning; matching hashes with an
/// inconsistent solve point at the repair side instead. Best-effort: any IO
/// error just truncates the dump.
async fn audit_staging_block_dump(dec: &mut EntryDecoder, sbn: u8) {
if std::env::var_os("ATP_RQ_INCONSISTENT_AUDIT").is_none() {
return;
}
let sbn_index = usize::from(sbn);
let Some(block) = dec.source_blocks.get(sbn_index) else {
return;
};
let (k, block_start, block_len) = (block.k, block.start, block.len);
let received: Vec<bool> = block.received.clone();
let seeded: Vec<bool> = block.pipeline_seeded.clone();
let Some(symbol_size) = dec
.pipeline
.as_ref()
.map(|pipeline| usize::from(pipeline.config_symbol_size()))
.filter(|size| *size > 0)
else {
return;
};
if flush_cached_entry_staging_file(dec).await.is_err() {
return;
}
let Ok(mut reader) = crate::fs::File::open(&dec.staging_path).await else {
return;
};
let Ok(absolute_block_start) = entry_staging_absolute_offset(dec, block_start, block_len)
else {
return;
};
let mut source_block = vec![0u8; block_len];
if reader
.seek(std::io::SeekFrom::Start(absolute_block_start))
.await
.is_err()
|| reader.read_exact(&mut source_block).await.is_err()
{
return;
}
eprintln!(
"[RQ_AUDIT] staging_dump entry={} sbn={sbn} k={k} block_len={block_len} abs_start={absolute_block_start}",
dec.index
);
// Hash the zero-PADDED symbol form (exactly how the seed path builds
// pipeline symbols) so staged hashes compare 1:1 with the decoder-side
// symbol dump, including the short final symbol.
let mut padded = vec![0u8; symbol_size];
for esi in 0..k {
let within = esi.saturating_mul(symbol_size);
if within >= block_len {
break;
}
let take = symbol_size.min(block_len - within);
padded.fill(0);
padded[..take].copy_from_slice(&source_block[within..within + take]);
eprintln!(
"[RQ_AUDIT] staged entry={} sbn={sbn} esi={esi} received={} seeded={} take={take} h8={:016x}",
dec.index,
received.get(esi).copied().unwrap_or(false),
seeded.get(esi).copied().unwrap_or(false),
crate::decoding::audit_fnv1a64(&padded),
);
}
}
async fn flush_cached_entry_staging_file(dec: &mut EntryDecoder) -> Result<(), RqError> {
if is_round_scoped_entry_staging_cache(dec) {
return close_cached_entry_staging_file(dec).await;
}
flush_source_write_buffer(dec).await?;
if let Some(file) = dec.staging_file.as_mut() {
file.flush().await?;
}
dec.staging_unflushed_bytes = 0;
Ok(())
}
async fn flush_cached_entry_staging_files(decoders: &mut [EntryDecoder]) -> Result<(), RqError> {
for dec in decoders {
flush_cached_entry_staging_file(dec).await?;
}
Ok(())
}
async fn write_entry_staging_range(
dec: &mut EntryDecoder,
offset: u64,
data: &[u8],
) -> Result<(), RqError> {
flush_source_write_buffer(dec).await?;
write_entry_staging_range_unbuffered(dec, offset, data).await
}
async fn write_source_staging_range(
dec: &mut EntryDecoder,
offset: u64,
data: &[u8],
) -> Result<(), RqError> {
if data.is_empty() {
return Ok(());
}
if !dec.cache_staging_file {
return write_entry_staging_range_unbuffered(dec, offset, data).await;
}
let contiguous = dec.source_write_buffer_offset.is_some_and(|buffer_offset| {
buffer_offset.checked_add(u64::try_from(dec.source_write_buffer.len()).unwrap_or(u64::MAX))
== Some(offset)
});
if !contiguous
|| dec.source_write_buffer.len().saturating_add(data.len()) > RQ_SOURCE_STAGE_BUFFER_BYTES
{
flush_source_write_buffer(dec).await?;
}
if data.len() >= RQ_SOURCE_STAGE_BUFFER_BYTES {
return write_entry_staging_range_unbuffered(dec, offset, data).await;
}
if dec.source_write_buffer.is_empty() {
dec.source_write_buffer_offset = Some(offset);
}
dec.source_write_buffer.extend_from_slice(data);
if dec.source_write_buffer.len() >= RQ_SOURCE_STAGE_BUFFER_BYTES {
flush_source_write_buffer(dec).await?;
}
Ok(())
}
async fn flush_source_write_buffer(dec: &mut EntryDecoder) -> Result<(), RqError> {
if dec.source_write_buffer.is_empty() {
dec.source_write_buffer_offset = None;
return Ok(());
}
let offset = dec.source_write_buffer_offset.take().ok_or_else(|| {
RqError::Coding(format!(
"entry {} source staging buffer missing offset",
dec.index
))
})?;
let mut buffered = Vec::new();
std::mem::swap(&mut buffered, &mut dec.source_write_buffer);
let result = write_entry_staging_range_unbuffered(dec, offset, &buffered).await;
buffered.clear();
dec.source_write_buffer = buffered;
result
}
async fn write_entry_staging_range_unbuffered(
dec: &mut EntryDecoder,
offset: u64,
data: &[u8],
) -> Result<(), RqError> {
let absolute_offset = entry_staging_absolute_offset(dec, offset, data.len())?;
if dec.cache_staging_file {
if dec.staging_file.is_none() {
let file = open_entry_staging_file(dec).await?;
dec.staging_file = Some(file);
dec.staging_cursor = None;
dec.staging_unflushed_bytes = 0;
}
let expected_cursor = dec.staging_cursor;
let next_cursor = absolute_offset
.checked_add(u64::try_from(data.len()).unwrap_or(u64::MAX))
.ok_or_else(|| {
RqError::Coding(format!("entry {} staging cursor overflow", dec.index))
})?;
let unflushed_bytes = dec.staging_unflushed_bytes.saturating_add(data.len());
let should_flush = unflushed_bytes >= RQ_SOURCE_STAGE_BUFFER_BYTES;
{
let file = dec
.staging_file
.as_mut()
.expect("staging file opened above");
if expected_cursor != Some(absolute_offset) {
file.seek(std::io::SeekFrom::Start(absolute_offset)).await?;
} else if staging_cursor_audit_enabled() {
// c54to7 diagnostic: validate the skip-seek invariant. The
// cached-handle fast path assumes the shared fd offset always
// equals the tracked cursor; a desync here silently lands
// bytes at the wrong staging offset (per-entry SHA mismatch
// at verify). Audit is env-gated (one extra lseek per
// skip-eligible write) and self-heals by re-seeking.
let actual = file.stream_position().await?;
if actual != absolute_offset {
rqtrace!(
"receiver: entry {} STAGING_CURSOR_DESYNC expected={} actual={} len={}",
dec.index,
absolute_offset,
actual,
data.len()
);
file.seek(std::io::SeekFrom::Start(absolute_offset)).await?;
}
}
file.write_all(data).await?;
if should_flush {
file.flush().await?;
}
}
dec.staging_cursor = Some(next_cursor);
dec.staging_unflushed_bytes = if should_flush { 0 } else { unflushed_bytes };
return Ok(());
}
let mut file = open_entry_staging_file(dec).await?;
file.seek(std::io::SeekFrom::Start(absolute_offset)).await?;
file.write_all(data).await?;
Ok(())
}
async fn create_receive_staging_dir(
dest_dir: &Path,
transfer_id: &str,
) -> Result<PathBuf, RqError> {
for _ in 0..RQ_STAGING_CREATE_ATTEMPTS {
let staging_seq = RQ_STAGING_SEQ.fetch_add(1, Ordering::Relaxed);
let staging_dir = dest_dir.join(format!(".atp-rq-staging-{transfer_id}-{staging_seq}"));
match crate::fs::create_dir(&staging_dir).await {
Ok(()) => return Ok(staging_dir),
Err(err) if err.kind() == std::io::ErrorKind::AlreadyExists => {}
Err(err) => return Err(RqError::Io(err)),
}
}
Err(RqError::Frame(format!(
"unable to create unique receiver staging directory for transfer {transfer_id}"
)))
}
// Reference full-check variant of the receiver symlink-prefix guard. All hot call
// sites now use `reject_destination_symlink_prefix_cached` (same fail-closed
// guarantee, per-prefix `lstat` caching), leaving this uncached form with no live
// caller. Kept as the canonical, cache-free reference implementation of the guard;
// per-fn `#[allow(dead_code)]` (asupersync convention) so `#![deny(dead_code)]`
// stays green without removing the primitive.
#[allow(dead_code)]
async fn reject_destination_symlink_prefix(base: &Path, out_path: &Path) -> Result<(), RqError> {
let rel = out_path.strip_prefix(base).map_err(|_| {
RqError::Source(format!(
"destination path {} is outside safe base {}",
out_path.display(),
base.display()
))
})?;
let mut current = base.to_path_buf();
reject_existing_symlink(¤t).await?;
for component in rel.components() {
let Component::Normal(component) = component else {
return Err(RqError::Source(format!(
"unsafe destination component in {}",
out_path.display()
)));
};
current.push(component);
reject_existing_symlink(¤t).await?;
}
Ok(())
}
/// Like [`reject_destination_symlink_prefix`] but skips path prefixes already proven non-symlink
/// in `verified`. During a single commit the receiver is the sole writer, so a prefix verified
/// non-symlink stays non-symlink; caching the per-component `lstat` across many files that share
/// parent directories turns O(files × depth) syscalls into O(unique prefixes). This is the
/// dominant receiver cost for many-small-file trees (tree_small `symlink_guard_micros`).
/// The per-component `Normal`-component safety check still runs for every path (it is a pure,
/// syscall-free check); only the redundant `lstat` of an already-verified prefix is elided, so
/// the fail-closed guarantee (any crossed symlink → error) is unchanged.
async fn reject_destination_symlink_prefix_cached(
base: &Path,
out_path: &Path,
verified: &mut BTreeSet<PathBuf>,
) -> Result<(), RqError> {
let rel = out_path.strip_prefix(base).map_err(|_| {
RqError::Source(format!(
"destination path {} is outside safe base {}",
out_path.display(),
base.display()
))
})?;
let mut current = base.to_path_buf();
if verified.insert(current.clone()) {
reject_existing_symlink(¤t).await?;
}
for component in rel.components() {
let Component::Normal(component) = component else {
return Err(RqError::Source(format!(
"unsafe destination component in {}",
out_path.display()
)));
};
current.push(component);
if verified.insert(current.clone()) {
reject_existing_symlink(¤t).await?;
}
}
Ok(())
}
async fn reject_existing_symlink(path: &Path) -> Result<(), RqError> {
match crate::fs::symlink_metadata(path).await {
Ok(metadata) if metadata.is_symlink() => Err(RqError::Source(format!(
"destination path crosses existing symlink: {}",
path.display()
))),
Ok(_) => Ok(()),
Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(()),
Err(err) => Err(RqError::Io(err)),
}
}
async fn persist_decoded_block(
dec: &mut EntryDecoder,
block_sbn: u8,
data: &[u8],
) -> Result<(), RqError> {
let block_size = u64::try_from(dec.max_block_size).map_err(|_| {
RqError::Coding(format!(
"entry {} max_block_size does not fit u64: {}",
dec.index, dec.max_block_size
))
})?;
let offset = u64::from(block_sbn)
.checked_mul(block_size)
.ok_or_else(|| RqError::Coding(format!("entry {} block offset overflow", dec.index)))?;
let end = offset
.checked_add(u64::try_from(data.len()).unwrap_or(u64::MAX))
.ok_or_else(|| RqError::Coding(format!("entry {} block end overflow", dec.index)))?;
if end > dec.size {
return Err(RqError::Frame(format!(
"decoded block {} for entry {} overruns declared size {}",
block_sbn, dec.index, dec.size
)));
}
write_entry_staging_range(dec, offset, data).await?;
// E-9 fix: `source_blocks[sbn].complete` is the single source of truth for both completion
// and byte accounting. A block decoded via FEC here can LATER also receive its final source
// symbols via retransmit; if we do not mark it complete now, `persist_source_symbol` would
// count this block's bytes a SECOND time (its `received_count == k` path), driving
// `bytes_written` past the entry size and causing `verify_and_commit` to FALSELY reject a
// byte-correct transfer as a "per-entry SHA-256 mismatch". Count the block exactly once and
// mark it done so any late source symbol for it is ignored by `persist_source_symbol`.
let block_idx = usize::from(block_sbn);
let already_complete = dec
.source_blocks
.get(block_idx)
.is_some_and(|block| block.complete);
if !already_complete {
dec.bytes_written = dec
.bytes_written
.checked_add(u64::try_from(data.len()).unwrap_or(u64::MAX))
.ok_or_else(|| RqError::Coding(format!("entry {} byte counter overflow", dec.index)))?;
}
if let Some(block) = dec.source_blocks.get_mut(block_idx) {
block.complete = true;
}
// When every source block is on disk (via source OR FEC) the entry is complete. This unifies
// the previously-desynced completion trackers (`source_blocks[].complete` vs
// `pipeline.is_complete()`) for the mixed source+FEC case, which is what NEITHER tracker fired
// for before (→ the bad-regime non-convergence). Empty `source_blocks` = non-source-streaming
// path, whose completion is still owned by `pipeline.is_complete()` at the call sites.
if source_streaming_entry_complete(dec) {
dec.complete = true;
dec.pipeline = None;
close_cached_entry_staging_file(dec).await?;
}
Ok(())
}
fn object_params_for(
object_id: ObjectId,
size: u64,
symbol_size: u16,
max_block_size: u64,
) -> ObjectParams {
let max_block = usize::try_from(max_block_size).unwrap_or(DEFAULT_MAX_BLOCK_SIZE);
let s = usize::from(symbol_size.max(1));
let total = usize::try_from(size).unwrap_or(0);
// Mirror the encoder's block plan: greedy max_block_size chunks.
let mut blocks = 0u16;
let mut max_k = 0usize;
if total > 0 {
let mut offset = 0usize;
while offset < total {
let len = (total - offset).min(max_block.max(1));
let k = len.div_ceil(s);
max_k = max_k.max(k);
blocks = blocks.saturating_add(1);
offset += len;
}
}
ObjectParams::new(
object_id,
size,
symbol_size,
blocks,
u16::try_from(max_k).unwrap_or(u16::MAX),
)
}
#[derive(Debug, Clone)]
struct LargeObjectCommitShard {
staging_path: PathBuf,
staging_offset: u64,
staging_shared: bool,
fragment: LargeObjectFragment,
}
#[derive(Debug, Clone)]
struct PackedMemberWrite {
offset: u64,
len: u64,
out_path: PathBuf,
}
async fn hash_packed_members_streaming(
staging_path: &Path,
members: &[PackedMember],
logical_digests: &mut Vec<EntryDigest>,
logical_files: &mut u64,
buf: &mut [u8],
) -> Result<bool, RqError> {
let mut file = crate::fs::File::open(staging_path)
.await
.map_err(|e| RqError::Source(format!("{}: {e}", staging_path.display())))?;
let mut cursor = 0u64;
let mut sha_ok = true;
for member in members {
let next_cursor = member.offset.checked_add(member.len).ok_or_else(|| {
RqError::Coding(format!(
"{}: packed member {} byte range overflows",
staging_path.display(),
member.rel_path
))
})?;
if cursor != member.offset {
file.seek(std::io::SeekFrom::Start(member.offset)).await?;
cursor = member.offset;
}
let mut sha = Sha256::new();
let mut content_id = crate::atp::object::ContentId::streaming();
let mut remaining = member.len;
while remaining > 0 {
let want = usize::try_from(remaining.min(buf.len() as u64)).unwrap_or(buf.len());
let n = file
.read(&mut buf[..want])
.await
.map_err(|e| RqError::Source(format!("{}: {e}", staging_path.display())))?;
if n == 0 {
return Err(RqError::Source(format!(
"{}: short read while verifying packed member {}",
staging_path.display(),
member.rel_path
)));
}
sha.update(&buf[..n]);
content_id.update(&buf[..n]);
remaining -= n as u64;
cursor = cursor.saturating_add(n as u64);
}
let member_sha: [u8; 32] = sha.finalize().into();
if hex_encode(&member_sha) != member.sha256_hex {
sha_ok = false;
}
logical_digests.push(EntryDigest {
rel_path: member.rel_path.clone(),
size: member.len,
content_id: crate::atp::object::ObjectId::content(content_id.finalize()),
content_sha256: member_sha,
});
*logical_files = (*logical_files).saturating_add(1);
cursor = next_cursor;
}
Ok(sha_ok)
}
/// One-shot packed-member commit cap: above this staged span the batch falls
/// back to the streaming cursor loop instead of materializing the span.
const PACKED_MEMBER_BATCH_ONESHOT_MAX_BYTES: u64 = 128 * 1024 * 1024;
/// Commit an entire packed small-file batch inside ONE blocking-pool task:
/// read the verified staged span once, then create/write every member with
/// raw `std::fs`. The serial async loop paid several pool round-trips per
/// tiny file (create + write + flush + staged reads ≈ thousands of
/// dispatches for a small tree), which is the commit_write tail rsync does
/// not pay (MATRIX-204/211). Parallel per-member writes buy little here —
/// same-directory creates serialize on the kernel dir lock — so the win is
/// eliminating dispatch overhead, not adding concurrency.
fn write_packed_member_batch_oneshot(
staging_path: PathBuf,
members: Vec<PackedMemberWrite>,
span_start: u64,
span_len: usize,
) -> std::io::Result<()> {
use std::io::{Read, Seek};
let mut staged = vec![0u8; span_len];
let mut source = std::fs::File::open(&staging_path)?;
source.seek(std::io::SeekFrom::Start(span_start))?;
source.read_exact(&mut staged)?;
let mut created_parents: BTreeSet<PathBuf> = BTreeSet::new();
for member in &members {
if let Some(parent) = member.out_path.parent()
&& created_parents.insert(parent.to_path_buf())
{
std::fs::create_dir_all(parent)?;
}
let start = usize::try_from(member.offset - span_start)
.map_err(|_| std::io::Error::other("packed member offset exceeds span"))?;
let len = usize::try_from(member.len)
.map_err(|_| std::io::Error::other("packed member length exceeds span"))?;
let end = start
.checked_add(len)
.filter(|end| *end <= staged.len())
.ok_or_else(|| std::io::Error::other("packed member range exceeds staged span"))?;
std::fs::write(&member.out_path, &staged[start..end])?;
}
Ok(())
}
async fn write_packed_member_batch(
staging_path: &Path,
members: &[PackedMemberWrite],
buf: &mut [u8],
) -> Result<(), RqError> {
let span_start = members.iter().map(|member| member.offset).min();
let span_end = members
.iter()
.map(|member| member.offset.saturating_add(member.len))
.max();
if let (Some(span_start), Some(span_end)) = (span_start, span_end)
&& members.len() > 1
&& span_end.saturating_sub(span_start) <= PACKED_MEMBER_BATCH_ONESHOT_MAX_BYTES
&& let Ok(span_len) = usize::try_from(span_end - span_start)
{
let staging = staging_path.to_path_buf();
let batch = members.to_vec();
let staging_display = staging_path.display().to_string();
return crate::runtime::spawn_blocking_io(move || {
write_packed_member_batch_oneshot(staging, batch, span_start, span_len)
})
.await
.map_err(|e| RqError::Source(format!("{staging_display}: {e}")));
}
let mut created_parents: BTreeSet<PathBuf> = BTreeSet::new();
for member in members {
if let Some(parent) = member.out_path.parent()
&& created_parents.insert(parent.to_path_buf())
{
crate::fs::create_dir_all(parent).await?;
}
}
let mut source = crate::fs::File::open(staging_path)
.await
.map_err(|e| RqError::Source(format!("{}: {e}", staging_path.display())))?;
let mut cursor = 0u64;
for member in members {
let next_cursor = member.offset.checked_add(member.len).ok_or_else(|| {
RqError::Coding(format!(
"{}: packed member destination {} byte range overflows",
staging_path.display(),
member.out_path.display()
))
})?;
if cursor != member.offset {
source.seek(std::io::SeekFrom::Start(member.offset)).await?;
cursor = member.offset;
}
let mut out = crate::fs::File::create(&member.out_path).await?;
let mut remaining = member.len;
while remaining > 0 {
let want = usize::try_from(remaining.min(buf.len() as u64)).unwrap_or(buf.len());
let n = source
.read(&mut buf[..want])
.await
.map_err(|e| RqError::Source(format!("{}: {e}", staging_path.display())))?;
if n == 0 {
return Err(RqError::Source(format!(
"{}: short read while committing packed member {}",
staging_path.display(),
member.out_path.display()
)));
}
out.write_all(&buf[..n]).await?;
remaining -= n as u64;
cursor = cursor.saturating_add(n as u64);
}
out.flush().await?;
cursor = next_cursor;
}
Ok(())
}
async fn hash_large_object_fragments(
shards: &[LargeObjectCommitShard],
buf: &mut [u8],
) -> Result<(u64, crate::atp::object::ObjectId, [u8; 32]), RqError> {
let mut sha = Sha256::new();
let mut cid = crate::atp::object::ContentId::streaming();
let mut total = 0u64;
for shard in shards {
let mut file = crate::fs::File::open(&shard.staging_path)
.await
.map_err(|e| RqError::Source(format!("{}: {e}", shard.staging_path.display())))?;
file.seek(std::io::SeekFrom::Start(shard.staging_offset))
.await
.map_err(|e| RqError::Source(format!("{}: {e}", shard.staging_path.display())))?;
let mut remaining = shard.fragment.len;
while remaining > 0 {
let want = usize::try_from(remaining.min(buf.len() as u64)).unwrap_or(buf.len());
let n = file
.read(&mut buf[..want])
.await
.map_err(|e| RqError::Source(format!("{}: {e}", shard.staging_path.display())))?;
if n == 0 {
return Err(RqError::Source(format!(
"{}: short read while hashing fragment {}",
shard.staging_path.display(),
shard.fragment.rel_path
)));
}
sha.update(&buf[..n]);
cid.update(&buf[..n]);
let n_u64 = n as u64;
remaining -= n_u64;
total = total.saturating_add(n_u64);
}
}
Ok((
total,
crate::atp::object::ObjectId::content(cid.finalize()),
sha.finalize().into(),
))
}
async fn write_large_object_fragments(
shards: &[LargeObjectCommitShard],
out_path: &Path,
buf: &mut [u8],
) -> Result<(), RqError> {
let mut out = crate::fs::File::create(out_path).await?;
for shard in shards {
let mut file = crate::fs::File::open(&shard.staging_path)
.await
.map_err(|e| RqError::Source(format!("{}: {e}", shard.staging_path.display())))?;
file.seek(std::io::SeekFrom::Start(shard.staging_offset))
.await
.map_err(|e| RqError::Source(format!("{}: {e}", shard.staging_path.display())))?;
let mut remaining = shard.fragment.len;
while remaining > 0 {
let want = usize::try_from(remaining.min(buf.len() as u64)).unwrap_or(buf.len());
let n = file
.read(&mut buf[..want])
.await
.map_err(|e| RqError::Source(format!("{}: {e}", shard.staging_path.display())))?;
if n == 0 {
return Err(RqError::Source(format!(
"{}: short read while committing fragment {}",
shard.staging_path.display(),
shard.fragment.rel_path
)));
}
out.write_all(&buf[..n]).await?;
remaining -= n as u64;
}
}
out.flush().await?;
Ok(())
}
fn contiguous_fragment_staging_path(shards: &[LargeObjectCommitShard]) -> Option<PathBuf> {
let first = shards.first()?;
if !first.staging_shared {
return None;
}
let staging_path = &first.staging_path;
let all_contiguous_shared = shards.iter().all(|shard| {
shard.staging_shared
&& shard.staging_path == *staging_path
&& shard.staging_offset == shard.fragment.logical_offset
});
all_contiguous_shared.then(|| staging_path.clone())
}
fn is_cross_device_rename_error(err: &std::io::Error) -> bool {
#[cfg(unix)]
{
const EXDEV_RAW_OS_ERROR: i32 = 18;
err.raw_os_error() == Some(EXDEV_RAW_OS_ERROR)
}
#[cfg(not(unix))]
{
let _ = err;
false
}
}
async fn remove_failed_fragment_staging_file(path: &Path) -> Result<(), RqError> {
match crate::fs::remove_file(path).await {
Ok(()) => Ok(()),
Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(()),
Err(err) => Err(RqError::Io(err)),
}
}
/// Verify every entry (SHA-256 + rebuilt merkle root) and, on success, atomically
/// write them to `dest_dir`.
///
/// E-15: an entry with non-empty `members` is a combined RaptorQ object; its
/// staging file is split into the member byte ranges, each member is verified
/// against its own SHA-256, and on commit the member files (not the packed
/// object) are written into place. The merkle root is rebuilt over the LOGICAL
/// files (members flattened), matching the sender's logical root. Verification is
/// fully separated from commit so a sha/merkle mismatch writes NOTHING.
async fn verify_and_commit(
manifest: &TransferManifest,
decoders: &mut [EntryDecoder],
dest_dir: &Path,
symbols_accepted: u64,
feedback_rounds: u32,
logical_precomputed: &BTreeMap<String, (u64, crate::atp::object::ObjectId, [u8; 32])>,
completion_digests: &CompletionDigestIndex,
) -> Result<ReceiveReceipt, RqError> {
let trace_commit = std::env::var_os("ATP_RQ_TRACE").is_some();
let total_started = trace_commit.then(Instant::now);
let mut close_flush_micros = 0u64;
let mut verify_hash_micros = 0u64;
let mut merkle_micros = 0u64;
let mut commit_plan_micros = 0u64;
let mut symlink_guard_micros = 0u64;
let mut commit_write_micros = 0u64;
for d in decoders.iter_mut() {
let close_started = trace_commit.then(Instant::now);
close_cached_entry_staging_file(d).await?;
if d.size == 0 && !d.staging_created {
let mut file = open_entry_staging_file(d).await?;
file.flush().await?;
}
close_flush_micros = close_flush_micros.saturating_add(elapsed_micros_since(close_started));
}
let mut sha_ok = true;
let mut received: u64 = 0;
// `logical_digests` holds one digest per LOGICAL file (members flattened) and
// drives the merkle check, matching the sender's logical root.
let mut logical_digests: Vec<EntryDigest> = Vec::with_capacity(manifest.entries.len());
// One commit plan per entry: rename the staging file (unpacked) or split it
// into member files (packed). Built only during verification; nothing is
// written until the sha+merkle gate passes.
enum EntryCommit {
/// Unpacked entry: rename its staging file to a single destination.
Rename {
rel_path: String,
staging_path: PathBuf,
},
/// Packed entry: split the staging file into member byte ranges.
Split {
staging_path: PathBuf,
members: Vec<PackedMember>,
},
/// Large-file entry split into ordered RaptorQ objects: reassemble the
/// shard staging files into one logical destination file.
Fragments {
rel_path: String,
shards: Vec<LargeObjectCommitShard>,
},
}
let mut commits: Vec<EntryCommit> = Vec::with_capacity(manifest.entries.len());
let mut fragment_groups: BTreeMap<String, Vec<LargeObjectCommitShard>> = BTreeMap::new();
let mut logical_files: u64 = 0;
let mut hash_buf = vec![0u8; RQ_STREAM_HASH_BUFFER_SIZE];
for e in &manifest.entries {
let Some(decoder) = decoders.iter().find(|d| d.index == e.index) else {
sha_ok = false;
continue;
};
// Gate on the CONTENT-ADDRESSED truth (actual file size + SHA-256, checked just below),
// NOT on the `bytes_written` side counter. `bytes_written` is kept for diagnostics but is
// not load-bearing for the commit decision: a byte-correct file with a transiently
// miscounted counter must commit (E-9 false-rejection). An incomplete or incorrect file is
// still rejected by the size+hash check that follows — that is the authoritative gate.
if !decoder.complete {
sha_ok = false;
}
// Object-level integrity: the staging file's size + SHA-256 must match the
// manifest entry. This applies to packed objects too (the concatenation).
// Fast path: the reliable control-source-stream already folded every
// in-order chunk into an incremental digest during receive, so reuse it
// instead of re-reading + re-hashing the whole staging file (the
// post-stream pass is otherwise the dominant clean-link tail). The lossy
// RaptorQ-datagram path leaves `inc_digest` None and re-hashes here.
let (size, content_id, content_sha256) =
if let Some((sz, cid, sha)) = decoder.inc_digest.as_ref() {
(*sz, cid.clone(), *sha)
} else if e.fragment.is_some() {
let hash_started = trace_commit.then(Instant::now);
let r = hash_file_range_streaming(
&decoder.staging_path,
decoder.staging_write_offset,
e.size,
&mut hash_buf,
)
.await
.map_err(|e| RqError::Source(e.into_message()))?;
verify_hash_micros =
verify_hash_micros.saturating_add(elapsed_micros_since(hash_started));
r
} else {
let hash_started = trace_commit.then(Instant::now);
let r = hash_file_streaming(&decoder.staging_path, &mut hash_buf)
.await
.map_err(|e| RqError::Source(e.into_message()))?;
verify_hash_micros =
verify_hash_micros.saturating_add(elapsed_micros_since(hash_started));
r
};
received = received.saturating_add(size);
let (expected_entry_size, expected_entry_sha256_hex) = completion_digests.expected_entry(e);
if size != e.size
|| size != expected_entry_size
|| hex_encode(&content_sha256) != expected_entry_sha256_hex
{
sha_ok = false;
}
if let Some(fragment) = &e.fragment {
fragment_groups
.entry(fragment.rel_path.clone())
.or_default()
.push(LargeObjectCommitShard {
staging_path: decoder.staging_path.clone(),
staging_offset: decoder.staging_write_offset,
staging_shared: decoder.staging_shared,
fragment: fragment.clone(),
});
} else if e.members.is_empty() {
// Normal single-file entry: its content IS the file (byte-identical to
// the prior wire). Its own digest is the logical digest; rename on commit.
logical_digests.push(EntryDigest {
rel_path: e.rel_path.clone(),
size,
content_id,
content_sha256,
});
logical_files = logical_files.saturating_add(1);
commits.push(EntryCommit::Rename {
rel_path: e.rel_path.clone(),
staging_path: decoder.staging_path.clone(),
});
} else {
// E-15 packed object: split the staging file into member byte ranges,
// verify each member's own SHA-256, and build a per-member logical
// digest. The packed object itself is not committed.
let hash_started = trace_commit.then(Instant::now);
sha_ok &= hash_packed_members_streaming(
&decoder.staging_path,
&e.members,
&mut logical_digests,
&mut logical_files,
&mut hash_buf,
)
.await?;
verify_hash_micros =
verify_hash_micros.saturating_add(elapsed_micros_since(hash_started));
commits.push(EntryCommit::Split {
staging_path: decoder.staging_path.clone(),
members: e.members.clone(),
});
}
}
for (rel_path, mut shards) in fragment_groups {
shards.sort_by_key(|shard| shard.fragment.shard_index);
// L2 fast path: if the reliable source-stream already folded every
// fragment (in arrival order) into a logical running hash, reuse it and
// skip re-reading + re-hashing all fragment staging files. The lossy
// datagram path leaves `logical_precomputed` empty and re-hashes here.
let (size, content_id, content_sha256) =
if let Some((sz, cid, sha)) = logical_precomputed.get(&rel_path) {
(*sz, cid.clone(), *sha)
} else {
let hash_started = trace_commit.then(Instant::now);
let r = hash_large_object_fragments(&shards, &mut hash_buf).await?;
verify_hash_micros =
verify_hash_micros.saturating_add(elapsed_micros_since(hash_started));
r
};
let Some(first) = shards.first() else {
sha_ok = false;
continue;
};
let (expected_logical_size, expected_logical_sha256_hex) = completion_digests
.expected_logical(
&rel_path,
first.fragment.logical_size,
&first.fragment.sha256_hex,
);
if size != first.fragment.logical_size
|| size != expected_logical_size
|| hex_encode(&content_sha256) != expected_logical_sha256_hex
{
sha_ok = false;
}
logical_digests.push(EntryDigest {
rel_path: rel_path.clone(),
size,
content_id,
content_sha256,
});
logical_files = logical_files.saturating_add(1);
commits.push(EntryCommit::Fragments { rel_path, shards });
}
let merkle_started = trace_commit.then(Instant::now);
let merkle_ok = flat_merkle_root_from_digests(&logical_digests)
== completion_digests.expected_merkle_root(manifest);
merkle_micros = merkle_micros.saturating_add(elapsed_micros_since(merkle_started));
let committed = sha_ok && merkle_ok;
let mut committed_paths: Vec<String> = Vec::new();
if !committed {
let mut cleaned_fragment_staging: BTreeSet<PathBuf> = BTreeSet::new();
for commit in &commits {
if let EntryCommit::Fragments { shards, .. } = commit
&& let Some(staging_path) = contiguous_fragment_staging_path(shards)
&& cleaned_fragment_staging.insert(staging_path.clone())
{
remove_failed_fragment_staging_file(&staging_path).await?;
}
}
}
if committed {
// `root_name` is attacker-controlled off the wire; collapse it to a
// single safe component so a hostile (absolute / separator-bearing)
// value cannot escape `dest_dir`.
let base = safe_base_for_root_name(dest_dir, &manifest.root_name)?;
// Resolve every LOGICAL destination path, rejecting any symlink prefix,
// before writing anything.
let plan_started = trace_commit.then(Instant::now);
enum CommitWrite {
Rename {
staging_path: PathBuf,
out_path: PathBuf,
},
Members {
staging_path: PathBuf,
members: Vec<PackedMemberWrite>,
},
Fragments {
shards: Vec<LargeObjectCommitShard>,
rename_staging_path: Option<PathBuf>,
out_path: PathBuf,
},
}
let mut writes: Vec<CommitWrite> = Vec::with_capacity(logical_digests.len());
for commit in &commits {
match commit {
EntryCommit::Rename {
rel_path,
staging_path,
} => {
let out_path = if manifest.is_directory {
join_relative(&base, rel_path)?
} else {
base.clone()
};
writes.push(CommitWrite::Rename {
staging_path: staging_path.clone(),
out_path,
});
}
EntryCommit::Split {
staging_path,
members,
} => {
// A packed object only ever occurs inside a directory transfer
// (the single-file path never packs), so members join under base.
let mut member_writes = Vec::with_capacity(members.len());
for member in members {
let out_path = join_relative(&base, &member.rel_path)?;
member_writes.push(PackedMemberWrite {
offset: member.offset,
len: member.len,
out_path,
});
}
writes.push(CommitWrite::Members {
staging_path: staging_path.clone(),
members: member_writes,
});
}
EntryCommit::Fragments { rel_path, shards } => {
let out_path = if manifest.is_directory {
join_relative(&base, rel_path)?
} else {
base.clone()
};
let rename_staging_path = contiguous_fragment_staging_path(shards);
writes.push(CommitWrite::Fragments {
shards: shards.clone(),
rename_staging_path,
out_path,
});
}
}
}
commit_plan_micros = commit_plan_micros.saturating_add(elapsed_micros_since(plan_started));
let symlink_started = trace_commit.then(Instant::now);
// Dedup the per-component symlink `lstat` across all destination paths: many files in a
// tree share parent directories, so verifying each unique prefix once (instead of per file)
// eliminates the redundant syscalls that dominate `symlink_guard_micros` for small-file
// trees. Fail-closed is preserved (every unique prefix + every final path is still checked).
let mut verified_prefixes: BTreeSet<PathBuf> = BTreeSet::new();
for write in &writes {
match write {
CommitWrite::Rename { out_path, .. } | CommitWrite::Fragments { out_path, .. } => {
reject_destination_symlink_prefix_cached(
&base,
out_path,
&mut verified_prefixes,
)
.await?;
}
CommitWrite::Members { members, .. } => {
for member in members {
reject_destination_symlink_prefix_cached(
&base,
&member.out_path,
&mut verified_prefixes,
)
.await?;
}
}
}
}
symlink_guard_micros =
symlink_guard_micros.saturating_add(elapsed_micros_since(symlink_started));
let write_started = trace_commit.then(Instant::now);
for write in writes {
match write {
CommitWrite::Rename {
staging_path,
out_path,
} => {
if let Some(parent) = out_path.parent() {
crate::fs::create_dir_all(parent).await?;
}
crate::fs::rename(&staging_path, &out_path).await?;
committed_paths.push(out_path.display().to_string());
}
CommitWrite::Members {
staging_path,
members,
} => {
// Re-read the verified member byte ranges from the packed
// staging file once, in offset order. This preserves the
// per-file outputs while avoiding one staging open/seek and
// one allocation per small tree file.
write_packed_member_batch(&staging_path, &members, &mut hash_buf).await?;
committed_paths.extend(
members
.into_iter()
.map(|member| member.out_path.display().to_string()),
);
}
CommitWrite::Fragments {
shards,
rename_staging_path,
out_path,
} => {
if let Some(parent) = out_path.parent() {
crate::fs::create_dir_all(parent).await?;
}
if let Some(staging_path) = rename_staging_path {
match crate::fs::rename(&staging_path, &out_path).await {
Ok(()) => {}
Err(err) if is_cross_device_rename_error(&err) => {
write_large_object_fragments(&shards, &out_path, &mut hash_buf)
.await?;
}
Err(err) => return Err(RqError::Io(err)),
}
} else {
write_large_object_fragments(&shards, &out_path, &mut hash_buf).await?;
}
committed_paths.push(out_path.display().to_string());
}
}
}
commit_write_micros =
commit_write_micros.saturating_add(elapsed_micros_since(write_started));
}
rqtrace!(
"receiver: verify_commit committed={} sha_ok={} merkle_ok={} bytes_received={} logical_files={} committed_paths={} feedback_rounds={} close_flush_micros={} verify_hash_micros={} merkle_micros={} commit_plan_micros={} symlink_guard_micros={} commit_write_micros={} total_micros={}",
committed,
sha_ok,
merkle_ok,
received,
logical_files,
committed_paths.len(),
feedback_rounds,
close_flush_micros,
verify_hash_micros,
merkle_micros,
commit_plan_micros,
symlink_guard_micros,
commit_write_micros,
elapsed_micros_since(total_started),
);
Ok(ReceiveReceipt {
committed,
bytes_received: received,
files: u32::try_from(logical_files).unwrap_or(u32::MAX),
sha_ok,
merkle_ok,
symbols_accepted,
feedback_rounds,
reason: if committed {
None
} else if !sha_ok {
Some("per-entry SHA-256 mismatch".to_string())
} else {
Some("merkle-root mismatch".to_string())
},
committed_paths,
})
}
fn parse_symbol_datagram_payload(
buf: &[u8],
n: usize,
tag: u64,
auth_required: bool,
) -> Option<(ParsedDatagram, &[u8])> {
let parsed = parse_symbol_header(&buf[..n], tag, auth_required)?;
let start = parsed.header_len;
let end = start + parsed.payload_len;
if end > n {
return None;
}
Some((parsed, &buf[start..end]))
}
#[derive(Debug, Default, Clone, Copy)]
struct RqDatagramIngest {
observed: bool,
accepted: bool,
source_observed: bool,
source_accepted: bool,
repair_observed: bool,
repair_accepted: bool,
payload_bytes: u64,
// Per-symbol receiver-intake stage timing. `feed_micros` is the legacy
// aggregate; the sub-stages below make large-lossy traces identify the
// exact feed bottleneck instead of blaming RaptorQ solve width.
parse_micros: u64,
feed_micros: u64,
source_auth_micros: u64,
source_persist_micros: u64,
pipeline_feed_micros: u64,
block_persist_micros: u64,
decode_dispatch_micros: u64,
source_seed_micros: u64,
feed_other_micros: u64,
decode_stats: RqDecodeRoundStats,
}
struct PlainSourceBatchSymbol<'a> {
decoder_index: usize,
sbn: usize,
esi: usize,
offset: u64,
take: usize,
payload: &'a [u8],
parse_micros: u64,
}
struct PlainSourceBatchRun {
decoder_index: usize,
sbn: usize,
first_esi: usize,
next_esi: usize,
offset: u64,
data: Vec<u8>,
symbols: u64,
payload_bytes: u64,
parse_micros: u64,
}
impl PlainSourceBatchRun {
fn new(symbol: PlainSourceBatchSymbol<'_>) -> Self {
let mut run = Self {
decoder_index: symbol.decoder_index,
sbn: symbol.sbn,
first_esi: symbol.esi,
next_esi: symbol.esi,
offset: symbol.offset,
data: Vec::with_capacity(symbol.take),
symbols: 0,
payload_bytes: 0,
parse_micros: 0,
};
run.absorb(symbol);
run
}
fn can_absorb(&self, symbol: &PlainSourceBatchSymbol<'_>) -> bool {
if self.decoder_index != symbol.decoder_index
|| self.sbn != symbol.sbn
|| self.next_esi != symbol.esi
{
return false;
}
self.offset
.checked_add(u64::try_from(self.data.len()).unwrap_or(u64::MAX))
== Some(symbol.offset)
}
fn absorb(&mut self, symbol: PlainSourceBatchSymbol<'_>) {
self.next_esi = symbol.esi.saturating_add(1);
self.symbols = self.symbols.saturating_add(1);
self.payload_bytes = self
.payload_bytes
.saturating_add(u64::try_from(symbol.payload.len()).unwrap_or(u64::MAX));
self.parse_micros = self.parse_micros.saturating_add(symbol.parse_micros);
self.data.extend_from_slice(&symbol.payload[..symbol.take]);
}
}
#[derive(Clone)]
struct AuthSourceBatchSymbol {
decoder_index: usize,
object_id: ObjectId,
sbn: usize,
sbn_wire: u8,
esi: usize,
esi_wire: u32,
offset: u64,
take: usize,
payload: Vec<u8>,
auth_tag: AuthenticationTag,
parse_micros: u64,
}
struct VerifiedAuthSourceBatchSymbol {
symbol: AuthSourceBatchSymbol,
verified: bool,
}
struct AuthSourceBatchRun {
decoder_index: usize,
sbn: usize,
first_esi: usize,
next_esi: usize,
offset: u64,
data: Vec<u8>,
auth_tags: Vec<AuthenticationTag>,
symbols: u64,
}
impl AuthSourceBatchRun {
fn new(symbol: AuthSourceBatchSymbol) -> Self {
let mut run = Self {
decoder_index: symbol.decoder_index,
sbn: symbol.sbn,
first_esi: symbol.esi,
next_esi: symbol.esi,
offset: symbol.offset,
data: Vec::with_capacity(symbol.take),
auth_tags: Vec::with_capacity(1),
symbols: 0,
};
run.absorb(symbol);
run
}
fn can_absorb(&self, symbol: &AuthSourceBatchSymbol) -> bool {
if self.decoder_index != symbol.decoder_index
|| self.sbn != symbol.sbn
|| self.next_esi != symbol.esi
{
return false;
}
self.offset
.checked_add(u64::try_from(self.data.len()).unwrap_or(u64::MAX))
== Some(symbol.offset)
}
fn absorb(&mut self, symbol: AuthSourceBatchSymbol) {
self.next_esi = symbol.esi.saturating_add(1);
self.symbols = self.symbols.saturating_add(1);
self.data.extend_from_slice(&symbol.payload[..symbol.take]);
self.auth_tags.push(symbol.auth_tag);
}
}
#[derive(Debug, Default, Clone, Copy)]
struct RqDecodeRoundStats {
attempts: u64,
repair_attempts: u64,
source_complete_attempts: u64,
completed_blocks: u64,
stale_requeues: u64,
decode_micros: u64,
join_wait_micros: u64,
apply_micros: u64,
persist_micros: u64,
queued_jobs: u64,
inline_jobs: u64,
spawn_denials: u64,
entry_cap_saturations: u64,
transfer_cap_saturations: u64,
pending_peak: u64,
}
impl RqDecodeRoundStats {
fn merge(&mut self, other: Self) {
self.attempts = self.attempts.saturating_add(other.attempts);
self.repair_attempts = self.repair_attempts.saturating_add(other.repair_attempts);
self.source_complete_attempts = self
.source_complete_attempts
.saturating_add(other.source_complete_attempts);
self.completed_blocks = self.completed_blocks.saturating_add(other.completed_blocks);
self.stale_requeues = self.stale_requeues.saturating_add(other.stale_requeues);
self.decode_micros = self.decode_micros.saturating_add(other.decode_micros);
self.join_wait_micros = self.join_wait_micros.saturating_add(other.join_wait_micros);
self.apply_micros = self.apply_micros.saturating_add(other.apply_micros);
self.persist_micros = self.persist_micros.saturating_add(other.persist_micros);
self.queued_jobs = self.queued_jobs.saturating_add(other.queued_jobs);
self.inline_jobs = self.inline_jobs.saturating_add(other.inline_jobs);
self.spawn_denials = self.spawn_denials.saturating_add(other.spawn_denials);
self.entry_cap_saturations = self
.entry_cap_saturations
.saturating_add(other.entry_cap_saturations);
self.transfer_cap_saturations = self
.transfer_cap_saturations
.saturating_add(other.transfer_cap_saturations);
self.pending_peak = self.pending_peak.max(other.pending_peak);
}
fn record_attempt(&mut self, kind: BlockDecodeKind, elapsed: Duration) {
self.attempts = self.attempts.saturating_add(1);
match kind {
BlockDecodeKind::SourceComplete => {
self.source_complete_attempts = self.source_complete_attempts.saturating_add(1);
}
BlockDecodeKind::RaptorQRepair => {
self.repair_attempts = self.repair_attempts.saturating_add(1);
}
}
self.decode_micros = self
.decode_micros
.saturating_add(duration_micros_saturating(elapsed));
}
fn record_join_wait(&mut self, elapsed: Duration) {
self.join_wait_micros = self
.join_wait_micros
.saturating_add(duration_micros_saturating(elapsed));
}
fn record_queued_job(&mut self, pending_jobs: usize) {
self.queued_jobs = self.queued_jobs.saturating_add(1);
self.pending_peak = self
.pending_peak
.max(u64::try_from(pending_jobs).unwrap_or(u64::MAX));
}
fn record_inline_job(&mut self) {
self.inline_jobs = self.inline_jobs.saturating_add(1);
}
fn record_spawn_denial(&mut self) {
self.spawn_denials = self.spawn_denials.saturating_add(1);
}
fn record_entry_cap_saturation(&mut self) {
self.entry_cap_saturations = self.entry_cap_saturations.saturating_add(1);
}
fn record_transfer_cap_saturation(&mut self) {
self.transfer_cap_saturations = self.transfer_cap_saturations.saturating_add(1);
}
}
fn trace_receiver_decode_profile(
phase: &str,
feedback_round: u32,
stats: RqDecodeRoundStats,
decode_width_budget: usize,
) {
rqtrace!(
"receiver: decode_profile phase={} feedback_round={} decode_attempts={} decode_repair_attempts={} decode_source_complete_attempts={} decode_completed_blocks={} decode_stale_requeues={} decode_micros={} decode_join_wait_micros={} decode_apply_micros={} decode_persist_micros={} decode_queued_jobs={} decode_inline_jobs={} decode_spawn_denials={} decode_entry_cap_saturations={} decode_transfer_cap_saturations={} decode_pending_peak={} decode_width_budget={}",
phase,
feedback_round,
stats.attempts,
stats.repair_attempts,
stats.source_complete_attempts,
stats.completed_blocks,
stats.stale_requeues,
stats.decode_micros,
stats.join_wait_micros,
stats.apply_micros,
stats.persist_micros,
stats.queued_jobs,
stats.inline_jobs,
stats.spawn_denials,
stats.entry_cap_saturations,
stats.transfer_cap_saturations,
stats.pending_peak,
decode_width_budget,
);
}
#[derive(Debug, Default, Clone, Copy)]
struct RqDatagramRoundStats {
observed: u64,
accepted: u64,
source_observed: u64,
source_accepted: u64,
repair_observed: u64,
repair_accepted: u64,
payload_bytes: u64,
// LEVER-R1 receiver-intake throughput instrumentation (sums over the round).
parse_micros: u64,
feed_micros: u64,
source_auth_micros: u64,
source_persist_micros: u64,
pipeline_feed_micros: u64,
block_persist_micros: u64,
decode_dispatch_micros: u64,
source_seed_micros: u64,
feed_other_micros: u64,
recv_micros: u64,
drain_micros: u64,
decode_stats: RqDecodeRoundStats,
}
impl RqDatagramRoundStats {
fn record(&mut self, ingest: RqDatagramIngest) {
if ingest.observed {
self.observed = self.observed.saturating_add(1);
self.payload_bytes = self.payload_bytes.saturating_add(ingest.payload_bytes);
}
if ingest.accepted {
self.accepted = self.accepted.saturating_add(1);
}
if ingest.source_observed {
self.source_observed = self.source_observed.saturating_add(1);
}
if ingest.source_accepted {
self.source_accepted = self.source_accepted.saturating_add(1);
}
if ingest.repair_observed {
self.repair_observed = self.repair_observed.saturating_add(1);
}
if ingest.repair_accepted {
self.repair_accepted = self.repair_accepted.saturating_add(1);
}
self.parse_micros = self.parse_micros.saturating_add(ingest.parse_micros);
self.feed_micros = self.feed_micros.saturating_add(ingest.feed_micros);
self.source_auth_micros = self
.source_auth_micros
.saturating_add(ingest.source_auth_micros);
self.source_persist_micros = self
.source_persist_micros
.saturating_add(ingest.source_persist_micros);
self.pipeline_feed_micros = self
.pipeline_feed_micros
.saturating_add(ingest.pipeline_feed_micros);
self.block_persist_micros = self
.block_persist_micros
.saturating_add(ingest.block_persist_micros);
self.decode_dispatch_micros = self
.decode_dispatch_micros
.saturating_add(ingest.decode_dispatch_micros);
self.source_seed_micros = self
.source_seed_micros
.saturating_add(ingest.source_seed_micros);
self.feed_other_micros = self
.feed_other_micros
.saturating_add(ingest.feed_other_micros);
self.decode_stats.merge(ingest.decode_stats);
}
fn merge(&mut self, other: Self) {
self.observed = self.observed.saturating_add(other.observed);
self.accepted = self.accepted.saturating_add(other.accepted);
self.source_observed = self.source_observed.saturating_add(other.source_observed);
self.source_accepted = self.source_accepted.saturating_add(other.source_accepted);
self.repair_observed = self.repair_observed.saturating_add(other.repair_observed);
self.repair_accepted = self.repair_accepted.saturating_add(other.repair_accepted);
self.payload_bytes = self.payload_bytes.saturating_add(other.payload_bytes);
self.parse_micros = self.parse_micros.saturating_add(other.parse_micros);
self.feed_micros = self.feed_micros.saturating_add(other.feed_micros);
self.source_auth_micros = self
.source_auth_micros
.saturating_add(other.source_auth_micros);
self.source_persist_micros = self
.source_persist_micros
.saturating_add(other.source_persist_micros);
self.pipeline_feed_micros = self
.pipeline_feed_micros
.saturating_add(other.pipeline_feed_micros);
self.block_persist_micros = self
.block_persist_micros
.saturating_add(other.block_persist_micros);
self.decode_dispatch_micros = self
.decode_dispatch_micros
.saturating_add(other.decode_dispatch_micros);
self.source_seed_micros = self
.source_seed_micros
.saturating_add(other.source_seed_micros);
self.feed_other_micros = self
.feed_other_micros
.saturating_add(other.feed_other_micros);
self.recv_micros = self.recv_micros.saturating_add(other.recv_micros);
self.drain_micros = self.drain_micros.saturating_add(other.drain_micros);
self.decode_stats.merge(other.decode_stats);
}
fn record_decode_stats(&mut self, decode_stats: RqDecodeRoundStats) {
self.decode_stats.merge(decode_stats);
}
fn record_recv_elapsed(&mut self, elapsed: Duration) {
self.recv_micros = self
.recv_micros
.saturating_add(duration_micros_saturating(elapsed));
}
fn record_tail_drain_elapsed(&mut self, elapsed: Duration) {
self.drain_micros = self
.drain_micros
.saturating_add(duration_micros_saturating(elapsed));
}
fn intake_micros(self) -> u64 {
self.parse_micros.saturating_add(self.feed_micros)
}
fn intake_symbols_per_s(self) -> u64 {
rate_per_second(self.observed, self.intake_micros())
}
fn intake_bytes_per_s(self) -> u64 {
rate_per_second(self.payload_bytes, self.intake_micros())
}
}
fn duration_micros_saturating(duration: Duration) -> u64 {
u64::try_from(duration.as_micros()).unwrap_or(u64::MAX)
}
fn elapsed_micros_since(started: Option<Instant>) -> u64 {
started.map_or(0, |instant| duration_micros_saturating(instant.elapsed()))
}
fn rate_per_second(units: u64, elapsed_micros: u64) -> u64 {
if elapsed_micros == 0 {
return 0;
}
let rate = u128::from(units).saturating_mul(1_000_000) / u128::from(elapsed_micros);
u64::try_from(rate).unwrap_or(u64::MAX)
}
fn rq_auth_verify_width_for_cx(cx: &Cx, symbols: usize) -> usize {
if symbols < RQ_AUTH_VERIFY_PARALLEL_MIN_SYMBOLS {
return 1;
}
if cx.blocking_pool_handle().is_none() {
return 1;
}
let chunks_by_size = symbols.div_ceil(RQ_AUTH_VERIFY_TARGET_CHUNK_SYMBOLS).max(1);
rq_decode_core_limit_for_cx(cx).min(chunks_by_size).max(1)
}
async fn feed_datagram_to_decoders(
cx: &Cx,
buf: &[u8],
n: usize,
tag: u64,
auth_required: bool,
symbol_auth: Option<&SecurityContext>,
decoders: &mut [EntryDecoder],
symbol_size: u16,
trace_intake: bool,
) -> Result<RqDatagramIngest, RqError> {
let parse_start = trace_intake.then(Instant::now);
let parsed_opt = parse_symbol_datagram_payload(buf, n, tag, auth_required);
let parse_micros = elapsed_micros_since(parse_start);
let Some((parsed, payload)) = parsed_opt else {
return Ok(RqDatagramIngest::default());
};
let Some(pos) = decoder_position_for_entry(decoders, parsed.entry) else {
return Ok(RqDatagramIngest::default());
};
let mut decode_stats = RqDecodeRoundStats::default();
let source_streaming_source = decoders[pos].source_streaming && parsed.kind.is_source();
let (allow_spawn_decode, decode_width_budget) = if source_streaming_source {
(false, 0)
} else {
let decode_width_budget = rq_decode_width_budget_for_cx(cx, decoders, symbol_size);
let mut pending_decode_jobs = rq_pending_decode_jobs(decoders);
if pending_decode_jobs >= decode_width_budget {
decode_stats
.merge(drain_ready_decodes(cx, decoders, false, decode_width_budget).await?);
pending_decode_jobs = rq_pending_decode_jobs(decoders);
}
(
pending_decode_jobs < decode_width_budget,
decode_width_budget,
)
};
let feed_start = trace_intake.then(Instant::now);
let feed = feed_symbol_with_cx(
cx,
&mut decoders[pos],
&parsed,
payload,
symbol_size,
symbol_auth,
allow_spawn_decode,
decode_width_budget,
trace_intake,
)
.await?;
decode_stats.merge(feed.decode_stats);
let feed_micros = elapsed_micros_since(feed_start);
let feed_accounted_micros = feed
.source_auth_micros
.saturating_add(feed.source_persist_micros)
.saturating_add(feed.pipeline_feed_micros)
.saturating_add(feed.block_persist_micros)
.saturating_add(feed.decode_dispatch_micros)
.saturating_add(feed.source_seed_micros);
let source_symbol = parsed.kind.is_source();
let repair_symbol = parsed.kind.is_repair();
Ok(RqDatagramIngest {
observed: true,
accepted: feed.accepted,
source_observed: source_symbol,
source_accepted: source_symbol && feed.accepted,
repair_observed: repair_symbol,
repair_accepted: repair_symbol && feed.accepted,
payload_bytes: u64::try_from(payload.len()).unwrap_or(u64::MAX),
parse_micros,
feed_micros,
source_auth_micros: feed.source_auth_micros,
source_persist_micros: feed.source_persist_micros,
pipeline_feed_micros: feed.pipeline_feed_micros,
block_persist_micros: feed.block_persist_micros,
decode_dispatch_micros: feed.decode_dispatch_micros,
source_seed_micros: feed.source_seed_micros,
feed_other_micros: feed_micros.saturating_sub(feed_accounted_micros),
decode_stats,
})
}
fn plain_source_batch_symbols<'a>(
batch: &'a crate::net::UdpRecvBatch,
tag: u64,
decoders: &[EntryDecoder],
symbol_size: u16,
trace_intake: bool,
) -> Option<Vec<PlainSourceBatchSymbol<'a>>> {
if batch.packets.is_empty() {
return Some(Vec::new());
}
let mut symbols = Vec::with_capacity(batch.packets.len());
let symbol_size = usize::from(symbol_size);
if symbol_size == 0 {
return None;
}
let mut seen = BTreeSet::new();
for packet in &batch.packets {
let parse_start = trace_intake.then(Instant::now);
let (parsed, payload) =
parse_symbol_datagram_payload(&packet.payload, packet.payload.len(), tag, false)?;
let parse_micros = elapsed_micros_since(parse_start);
if !parsed.kind.is_source() || payload.len() != symbol_size {
return None;
}
let decoder_index = decoder_position_for_entry(decoders, parsed.entry)?;
let decoder = &decoders[decoder_index];
if decoder.complete || !decoder.source_streaming {
return None;
}
let sbn = usize::from(parsed.sbn);
let esi = usize::try_from(parsed.esi).ok()?;
if !seen.insert((decoder_index, sbn, esi)) {
return None;
}
let within_block = esi.checked_mul(symbol_size)?;
let block = decoder.source_blocks.get(sbn)?;
if block.complete || esi >= block.k || block.received[esi] || within_block >= block.len {
return None;
}
let take = symbol_size.min(block.len - within_block);
let offset = block.start.checked_add(u64::try_from(within_block).ok()?)?;
symbols.push(PlainSourceBatchSymbol {
decoder_index,
sbn,
esi,
offset,
take,
payload,
parse_micros,
});
}
Some(symbols)
}
async fn persist_plain_source_batch_run(
dec: &mut EntryDecoder,
run: &PlainSourceBatchRun,
) -> Result<u64, RqError> {
if run.symbols == 0 {
return Ok(0);
}
let Some(block) = dec.source_blocks.get(run.sbn) else {
return Ok(0);
};
if block.complete || run.next_esi > block.k {
return Ok(0);
}
for esi in run.first_esi..run.next_esi {
if block.received[esi] {
return Ok(0);
}
}
write_source_staging_range(dec, run.offset, &run.data).await?;
let completed_now = {
let block = &mut dec.source_blocks[run.sbn];
for esi in run.first_esi..run.next_esi {
if block.received[esi] {
continue;
}
block.received[esi] = true;
block.auth_tags[esi] = None;
block.received_count = block.received_count.saturating_add(1);
}
if block.received_count == block.k {
block.complete = true;
dec.bytes_written = dec
.bytes_written
.checked_add(u64::try_from(block.len).unwrap_or(u64::MAX))
.ok_or_else(|| {
RqError::Coding(format!("entry {} byte counter overflow", dec.index))
})?;
true
} else {
false
}
};
if completed_now {
rqtrace!(
"receiver: entry {} completed source-streamed block {} from plain-source batch",
dec.index,
run.sbn
);
}
if source_streaming_entry_complete(dec) {
dec.complete = true;
dec.pipeline = None;
close_cached_entry_staging_file(dec).await?;
}
Ok(run.symbols)
}
async fn flush_plain_source_batch_run(
stats: &mut RqDatagramRoundStats,
decoders: &mut [EntryDecoder],
run: Option<PlainSourceBatchRun>,
trace_intake: bool,
) -> Result<(), RqError> {
let Some(run) = run else {
return Ok(());
};
let persist_start = trace_intake.then(Instant::now);
let accepted = persist_plain_source_batch_run(&mut decoders[run.decoder_index], &run).await?;
let persist_micros = elapsed_micros_since(persist_start);
stats.observed = stats.observed.saturating_add(run.symbols);
stats.accepted = stats.accepted.saturating_add(accepted);
stats.source_observed = stats.source_observed.saturating_add(run.symbols);
stats.source_accepted = stats.source_accepted.saturating_add(accepted);
stats.payload_bytes = stats.payload_bytes.saturating_add(run.payload_bytes);
stats.parse_micros = stats.parse_micros.saturating_add(run.parse_micros);
stats.feed_micros = stats.feed_micros.saturating_add(persist_micros);
stats.source_persist_micros = stats.source_persist_micros.saturating_add(persist_micros);
Ok(())
}
async fn try_feed_plain_source_datagram_batch(
batch: &crate::net::UdpRecvBatch,
tag: u64,
decoders: &mut [EntryDecoder],
symbol_size: u16,
trace_intake: bool,
) -> Option<Result<RqDatagramRoundStats, RqError>> {
let symbols = plain_source_batch_symbols(batch, tag, decoders, symbol_size, trace_intake)?;
let mut stats = RqDatagramRoundStats::default();
let mut run = None;
for symbol in symbols {
let starts_new_run = run
.as_ref()
.is_some_and(|active: &PlainSourceBatchRun| !active.can_absorb(&symbol));
if starts_new_run
&& let Err(err) =
flush_plain_source_batch_run(&mut stats, decoders, run.take(), trace_intake).await
{
return Some(Err(err));
}
if let Some(active) = run.as_mut() {
active.absorb(symbol);
} else {
run = Some(PlainSourceBatchRun::new(symbol));
}
}
if let Err(err) = flush_plain_source_batch_run(&mut stats, decoders, run, trace_intake).await {
return Some(Err(err));
}
Some(Ok(stats))
}
fn authenticated_source_batch_symbols(
batch: &crate::net::UdpRecvBatch,
tag: u64,
decoders: &[EntryDecoder],
symbol_size: u16,
trace_intake: bool,
) -> Option<Vec<AuthSourceBatchSymbol>> {
let mut symbols = Vec::with_capacity(batch.packets.len());
let symbol_size = usize::from(symbol_size);
if symbol_size == 0 {
return None;
}
let mut seen = BTreeSet::new();
for packet in &batch.packets {
let parse_start = trace_intake.then(Instant::now);
let (parsed, payload) =
parse_symbol_datagram_payload(&packet.payload, packet.payload.len(), tag, true)?;
let parse_micros = elapsed_micros_since(parse_start);
if !parsed.kind.is_source() || payload.len() != symbol_size {
return None;
}
let auth_tag = parsed.auth_tag?;
let decoder_index = decoder_position_for_entry(decoders, parsed.entry)?;
let decoder = &decoders[decoder_index];
if decoder.complete || !decoder.source_streaming {
return None;
}
let sbn = usize::from(parsed.sbn);
let esi = usize::try_from(parsed.esi).ok()?;
if !seen.insert((decoder_index, sbn, esi)) {
return None;
}
let within_block = esi.checked_mul(symbol_size)?;
let block = decoder.source_blocks.get(sbn)?;
if block.complete || esi >= block.k || block.received[esi] || within_block >= block.len {
return None;
}
let take = symbol_size.min(block.len - within_block);
let offset = block.start.checked_add(u64::try_from(within_block).ok()?)?;
symbols.push(AuthSourceBatchSymbol {
decoder_index,
object_id: decoder.object_id,
sbn,
sbn_wire: parsed.sbn,
esi,
esi_wire: parsed.esi,
offset,
take,
payload: payload.to_vec(),
auth_tag,
parse_micros,
});
}
Some(symbols)
}
fn verify_auth_source_batch_chunk(
context: SecurityContext,
symbols: Vec<AuthSourceBatchSymbol>,
) -> Vec<VerifiedAuthSourceBatchSymbol> {
symbols
.into_iter()
.map(|mut symbol| {
let auth_symbol = Symbol::new(
SymbolId::new(symbol.object_id, symbol.sbn_wire, symbol.esi_wire),
symbol.payload.clone(),
SymbolKind::Source,
);
let mut auth = AuthenticatedSymbol::from_parts(auth_symbol, symbol.auth_tag);
let verified =
context.verify_authenticated_symbol(&mut auth).is_ok() && auth.is_verified();
if verified {
symbol.payload = auth.into_symbol().into_data();
}
VerifiedAuthSourceBatchSymbol { symbol, verified }
})
.collect()
}
async fn verify_auth_source_batch_symbols(
cx: &Cx,
context: &SecurityContext,
symbols: Vec<AuthSourceBatchSymbol>,
trace_intake: bool,
) -> Result<(Vec<VerifiedAuthSourceBatchSymbol>, u64), RqError> {
let auth_start = trace_intake.then(Instant::now);
let width = rq_auth_verify_width_for_cx(cx, symbols.len());
if width <= 1 {
return Ok((
verify_auth_source_batch_chunk(context.clone(), symbols),
elapsed_micros_since(auth_start),
));
}
let chunk_len = symbols
.len()
.div_ceil(width)
.max(RQ_AUTH_VERIFY_TARGET_CHUNK_SYMBOLS);
let mut pending = Vec::new();
for chunk in symbols.chunks(chunk_len) {
let chunk = chunk.to_vec();
let inline_chunk = chunk.clone();
let spawn_context = context.clone();
let inline_context = context.clone();
match cx.spawn_blocking(move |_child| verify_auth_source_batch_chunk(spawn_context, chunk))
{
Ok(handle) => pending.push(Ok(handle)),
Err(_) => pending.push(Err(verify_auth_source_batch_chunk(
inline_context,
inline_chunk,
))),
}
}
let mut verified = Vec::new();
for pending_chunk in pending {
match pending_chunk {
Ok(mut handle) => verified.extend(handle.join(cx).await.map_err(|join_err| {
RqError::Authentication(format!(
"RQ source auth verification worker failed: {join_err:?}"
))
})?),
Err(mut inline) => verified.append(&mut inline),
}
}
Ok((verified, elapsed_micros_since(auth_start)))
}
async fn persist_auth_source_batch_run(
dec: &mut EntryDecoder,
run: &AuthSourceBatchRun,
) -> Result<u64, RqError> {
if run.symbols == 0 {
return Ok(0);
}
let Some(block) = dec.source_blocks.get(run.sbn) else {
return Ok(0);
};
if block.complete || run.next_esi > block.k {
return Ok(0);
}
let expected_tags = run.next_esi.saturating_sub(run.first_esi);
if run.auth_tags.len() != expected_tags {
return Err(RqError::Coding(format!(
"entry {} authenticated source batch tag count mismatch: have {} expected {}",
dec.index,
run.auth_tags.len(),
expected_tags
)));
}
for esi in run.first_esi..run.next_esi {
if block.received[esi] {
return Ok(0);
}
}
write_source_staging_range(dec, run.offset, &run.data).await?;
let completed_now = {
let block = &mut dec.source_blocks[run.sbn];
for (tag_index, esi) in (run.first_esi..run.next_esi).enumerate() {
if block.received[esi] {
continue;
}
block.received[esi] = true;
block.auth_tags[esi] = Some(run.auth_tags[tag_index]);
block.received_count = block.received_count.saturating_add(1);
}
if block.received_count == block.k {
block.complete = true;
dec.bytes_written = dec
.bytes_written
.checked_add(u64::try_from(block.len).unwrap_or(u64::MAX))
.ok_or_else(|| {
RqError::Coding(format!("entry {} byte counter overflow", dec.index))
})?;
true
} else {
false
}
};
if completed_now {
rqtrace!(
"receiver: entry {} completed source-streamed block {} from auth-source batch",
dec.index,
run.sbn
);
}
if source_streaming_entry_complete(dec) {
dec.complete = true;
dec.pipeline = None;
close_cached_entry_staging_file(dec).await?;
}
Ok(run.symbols)
}
async fn flush_auth_source_batch_run(
stats: &mut RqDatagramRoundStats,
decoders: &mut [EntryDecoder],
run: Option<AuthSourceBatchRun>,
trace_intake: bool,
) -> Result<(), RqError> {
let Some(run) = run else {
return Ok(());
};
let persist_start = trace_intake.then(Instant::now);
let accepted = persist_auth_source_batch_run(&mut decoders[run.decoder_index], &run).await?;
let persist_micros = elapsed_micros_since(persist_start);
stats.accepted = stats.accepted.saturating_add(accepted);
stats.source_accepted = stats.source_accepted.saturating_add(accepted);
stats.feed_micros = stats.feed_micros.saturating_add(persist_micros);
stats.source_persist_micros = stats.source_persist_micros.saturating_add(persist_micros);
Ok(())
}
async fn try_feed_authenticated_source_datagram_batch(
cx: &Cx,
batch: &crate::net::UdpRecvBatch,
tag: u64,
context: &SecurityContext,
decoders: &mut [EntryDecoder],
symbol_size: u16,
trace_intake: bool,
) -> Option<Result<RqDatagramRoundStats, RqError>> {
if context.mode() == AuthMode::Disabled {
return None;
}
let symbols =
authenticated_source_batch_symbols(batch, tag, decoders, symbol_size, trace_intake)?;
let mut stats = RqDatagramRoundStats::default();
for symbol in &symbols {
stats.observed = stats.observed.saturating_add(1);
stats.source_observed = stats.source_observed.saturating_add(1);
stats.payload_bytes = stats
.payload_bytes
.saturating_add(u64::try_from(symbol.payload.len()).unwrap_or(u64::MAX));
stats.parse_micros = stats.parse_micros.saturating_add(symbol.parse_micros);
}
let (verified, auth_micros) =
match verify_auth_source_batch_symbols(cx, context, symbols, trace_intake).await {
Ok(verified) => verified,
Err(err) => return Some(Err(err)),
};
stats.source_auth_micros = stats.source_auth_micros.saturating_add(auth_micros);
stats.feed_micros = stats.feed_micros.saturating_add(auth_micros);
let mut run = None;
for verified_symbol in verified {
let symbol = verified_symbol.symbol;
if !verified_symbol.verified {
rqtrace!(
"receiver: entry {} rejected source-streamed batch sbn={} esi={} auth tag",
decoders[symbol.decoder_index].index,
symbol.sbn,
symbol.esi
);
continue;
}
let starts_new_run = run
.as_ref()
.is_some_and(|active: &AuthSourceBatchRun| !active.can_absorb(&symbol));
if starts_new_run
&& let Err(err) =
flush_auth_source_batch_run(&mut stats, decoders, run.take(), trace_intake).await
{
return Some(Err(err));
}
if let Some(active) = run.as_mut() {
active.absorb(symbol);
} else {
run = Some(AuthSourceBatchRun::new(symbol));
}
}
if let Err(err) = flush_auth_source_batch_run(&mut stats, decoders, run, trace_intake).await {
return Some(Err(err));
}
Some(Ok(stats))
}
async fn feed_datagram_batch_to_decoders(
cx: &Cx,
batch: &crate::net::UdpRecvBatch,
tag: u64,
auth_required: bool,
symbol_auth: Option<&SecurityContext>,
decoders: &mut [EntryDecoder],
symbol_size: u16,
trace_intake: bool,
) -> Result<RqDatagramRoundStats, RqError> {
if !auth_required
&& symbol_auth.is_none()
&& rq_pending_decode_jobs(decoders) == 0
&& let Some(result) =
try_feed_plain_source_datagram_batch(batch, tag, decoders, symbol_size, trace_intake)
.await
{
let mut stats = result?;
stats.record_decode_stats(drain_ready_decodes_if_pending(cx, decoders, symbol_size).await?);
return Ok(stats);
}
if auth_required
&& rq_pending_decode_jobs(decoders) == 0
&& let Some(context) = symbol_auth
&& let Some(result) = try_feed_authenticated_source_datagram_batch(
cx,
batch,
tag,
context,
decoders,
symbol_size,
trace_intake,
)
.await
{
let mut stats = result?;
stats.record_decode_stats(drain_ready_decodes_if_pending(cx, decoders, symbol_size).await?);
return Ok(stats);
}
let mut stats = RqDatagramRoundStats::default();
for packet in &batch.packets {
stats.record(
feed_datagram_to_decoders(
cx,
&packet.payload,
packet.payload.len(),
tag,
auth_required,
symbol_auth,
decoders,
symbol_size,
trace_intake,
)
.await?,
);
}
stats.record_decode_stats(drain_ready_decodes_if_pending(cx, decoders, symbol_size).await?);
Ok(stats)
}
async fn drain_ready_decodes_if_pending(
cx: &Cx,
decoders: &mut [EntryDecoder],
symbol_size: u16,
) -> Result<RqDecodeRoundStats, RqError> {
if rq_pending_decode_jobs(decoders) == 0 {
return Ok(RqDecodeRoundStats::default());
}
let decode_width_budget = rq_decode_width_budget_for_cx(cx, decoders, symbol_size);
drain_ready_decodes(cx, decoders, true, decode_width_budget).await
}
#[derive(Debug, Default, Clone, Copy)]
struct DecodeApplyOutcome {
completed: bool,
persist_micros: u64,
}
async fn apply_decode_result(
dec: &mut EntryDecoder,
result: SymbolAcceptResult,
decode_elapsed: Duration,
) -> Result<DecodeApplyOutcome, RqError> {
let decode_micros = duration_micros_saturating(decode_elapsed);
match result {
SymbolAcceptResult::BlockComplete { block_sbn, data } => {
let persist_start = Instant::now();
persist_decoded_block(dec, block_sbn, &data).await?;
let persist_micros = duration_micros_saturating(persist_start.elapsed());
if dec.complete
|| dec
.pipeline
.as_ref()
.is_some_and(DecodingPipeline::is_complete)
{
dec.complete = true;
dec.pipeline = None;
}
rqtrace!(
"receiver: entry {} completed parallel decode block {} decode_micros={}",
dec.index,
block_sbn,
decode_micros
);
Ok(DecodeApplyOutcome {
completed: true,
persist_micros,
})
}
SymbolAcceptResult::Rejected(reason) => {
rqtrace!(
"receiver: entry {} parallel decode rejected reason={reason:?} decode_micros={}",
dec.index,
decode_micros
);
Ok(DecodeApplyOutcome::default())
}
SymbolAcceptResult::Accepted { .. }
| SymbolAcceptResult::DecodingStarted { .. }
| SymbolAcceptResult::Duplicate => Ok(DecodeApplyOutcome::default()),
}
}
async fn finalize_decode_outcome(
cx: &Cx,
dec: &mut EntryDecoder,
outcome: BlockDecodeOutcome,
allow_spawn_decode: bool,
transfer_decode_width: usize,
) -> Result<RqDecodeRoundStats, RqError> {
let mut stats = RqDecodeRoundStats::default();
let decode_elapsed = outcome.elapsed();
let outcome_sbn = outcome.sbn();
stats.record_attempt(outcome.kind(), decode_elapsed);
let Some(pipeline) = dec.pipeline.as_mut() else {
return Ok(stats);
};
let apply_start = Instant::now();
match pipeline.finish_decode_job_deferred(outcome) {
DeferredSymbolAcceptResult::Immediate(result) => {
let audit_inconsistent = matches!(
&result,
SymbolAcceptResult::Rejected(RejectReason::InconsistentEquations)
);
let applied = apply_decode_result(dec, result, decode_elapsed).await?;
if audit_inconsistent {
audit_staging_block_dump(dec, outcome_sbn).await;
}
stats.apply_micros = stats
.apply_micros
.saturating_add(duration_micros_saturating(apply_start.elapsed()));
stats.persist_micros = stats.persist_micros.saturating_add(applied.persist_micros);
if applied.completed {
stats.completed_blocks = stats.completed_blocks.saturating_add(1);
}
}
DeferredSymbolAcceptResult::Decode(job) => {
let block_sbn = job.sbn();
let decode_micros = duration_micros_saturating(decode_elapsed);
stats.apply_micros = stats
.apply_micros
.saturating_add(duration_micros_saturating(apply_start.elapsed()));
rqtrace!(
"receiver: entry {} requeued stale parallel decode block {} decode_micros={}",
dec.index,
block_sbn,
decode_micros
);
stats.stale_requeues = stats.stale_requeues.saturating_add(1);
stats.merge(
queue_stale_decode_retry(cx, dec, job, allow_spawn_decode, transfer_decode_width)
.await?,
);
}
}
Ok(stats)
}
async fn queue_stale_decode_retry(
cx: &Cx,
dec: &mut EntryDecoder,
job: BlockDecodeJob,
allow_spawn_decode: bool,
transfer_decode_width: usize,
) -> Result<RqDecodeRoundStats, RqError> {
let mut stats = RqDecodeRoundStats::default();
let block_sbn = job.sbn();
if dec.complete || dec.pipeline.is_none() {
return Ok(stats);
}
if block_decode_pending(dec, block_sbn) {
if let Some(pipeline) = dec.pipeline.as_mut() {
pipeline.restore_decode_job(job);
}
return Ok(stats);
}
let entry_decode_width = entry_decode_width_budget(dec, transfer_decode_width);
if entry_decode_width <= 1 {
let outcome = run_block_decode_job(job);
let decode_elapsed = outcome.elapsed();
stats.record_attempt(outcome.kind(), decode_elapsed);
let Some(pipeline) = dec.pipeline.as_mut() else {
return Ok(stats);
};
let apply_start = Instant::now();
let result = pipeline.finish_decode_job(outcome);
let applied = apply_decode_result(dec, result, decode_elapsed).await?;
stats.apply_micros = stats
.apply_micros
.saturating_add(duration_micros_saturating(apply_start.elapsed()));
stats.persist_micros = stats.persist_micros.saturating_add(applied.persist_micros);
stats.record_inline_job();
if applied.completed {
stats.completed_blocks = stats.completed_blocks.saturating_add(1);
}
rqtrace!(
"receiver: entry {} ran stale decode retry block {} inline because entry size/block-count is below the parallel decode gate",
dec.index,
block_sbn
);
return Ok(stats);
}
if !allow_spawn_decode
|| !can_spawn_parallel_decode(dec.pending_decodes.len(), entry_decode_width)
{
if !allow_spawn_decode {
stats.record_transfer_cap_saturation();
} else {
stats.record_entry_cap_saturation();
}
if let Some(pipeline) = dec.pipeline.as_mut() {
pipeline.restore_decode_job(job);
}
rqtrace!(
"receiver: entry {} deferred stale decode retry for block {} because decode width is saturated (entry_cap={entry_decode_width})",
dec.index,
block_sbn
);
return Ok(stats);
}
let inline_job = job.clone();
match cx.spawn_blocking(move |_child| run_block_decode_job(job)) {
Ok(handle) => {
dec.pending_decodes
.push(PendingDecode { block_sbn, handle });
stats.record_queued_job(dec.pending_decodes.len());
Ok(stats)
}
Err(crate::runtime::state::SpawnError::RuntimeUnavailable) => {
stats.record_spawn_denial();
let outcome = run_block_decode_job(inline_job);
let decode_elapsed = outcome.elapsed();
stats.record_attempt(outcome.kind(), decode_elapsed);
let Some(pipeline) = dec.pipeline.as_mut() else {
return Ok(stats);
};
let apply_start = Instant::now();
let result = pipeline.finish_decode_job(outcome);
let applied = apply_decode_result(dec, result, decode_elapsed).await?;
stats.apply_micros = stats
.apply_micros
.saturating_add(duration_micros_saturating(apply_start.elapsed()));
stats.persist_micros = stats.persist_micros.saturating_add(applied.persist_micros);
stats.record_inline_job();
if applied.completed {
stats.completed_blocks = stats.completed_blocks.saturating_add(1);
}
Ok(stats)
}
Err(err) => {
stats.record_spawn_denial();
if let Some(pipeline) = dec.pipeline.as_mut() {
pipeline.restore_decode_job(inline_job);
}
rqtrace!(
"receiver: entry {} deferred stale decode retry for block {} after spawn denial: {err:?}",
dec.index,
block_sbn
);
Ok(stats)
}
}
}
async fn drain_ready_decodes(
cx: &Cx,
decoders: &mut [EntryDecoder],
allow_spawn_decode: bool,
transfer_decode_width: usize,
) -> Result<RqDecodeRoundStats, RqError> {
let mut stats = RqDecodeRoundStats::default();
for dec in decoders {
stats.merge(
drain_ready_entry_decodes(cx, dec, allow_spawn_decode, transfer_decode_width).await?,
);
}
Ok(stats)
}
async fn join_all_pending_decodes(
cx: &Cx,
decoders: &mut [EntryDecoder],
transfer_decode_width: usize,
) -> Result<RqDecodeRoundStats, RqError> {
let mut stats = RqDecodeRoundStats::default();
for dec in decoders {
while let Some(mut pending) = dec.pending_decodes.pop() {
stats.merge(
join_pending_decode(cx, dec, &mut pending, true, transfer_decode_width).await?,
);
}
}
Ok(stats)
}
async fn drain_ready_entry_decodes(
cx: &Cx,
dec: &mut EntryDecoder,
allow_spawn_decode: bool,
transfer_decode_width: usize,
) -> Result<RqDecodeRoundStats, RqError> {
let mut stats = RqDecodeRoundStats::default();
let mut i = 0usize;
while i < dec.pending_decodes.len() {
if !dec.pending_decodes[i].handle.is_finished() {
i += 1;
continue;
}
let mut pending = dec.pending_decodes.swap_remove(i);
stats.merge(
join_pending_decode(
cx,
dec,
&mut pending,
allow_spawn_decode,
transfer_decode_width,
)
.await?,
);
}
Ok(stats)
}
async fn join_one_pending_decode(
cx: &Cx,
dec: &mut EntryDecoder,
allow_spawn_decode: bool,
transfer_decode_width: usize,
) -> Result<RqDecodeRoundStats, RqError> {
let Some(mut pending) = dec.pending_decodes.pop() else {
return Ok(RqDecodeRoundStats::default());
};
join_pending_decode(
cx,
dec,
&mut pending,
allow_spawn_decode,
transfer_decode_width,
)
.await
}
async fn join_pending_decode(
cx: &Cx,
dec: &mut EntryDecoder,
pending: &mut PendingDecode,
allow_spawn_decode: bool,
transfer_decode_width: usize,
) -> Result<RqDecodeRoundStats, RqError> {
let block_sbn = pending.block_sbn;
let join_start = Instant::now();
let outcome = pending.handle.join(cx).await.map_err(|join_err| {
RqError::Coding(format!(
"decode task failed for entry {} block {}: {join_err:?}",
dec.index, block_sbn
))
})?;
let join_wait = join_start.elapsed();
let mut stats =
finalize_decode_outcome(cx, dec, outcome, allow_spawn_decode, transfer_decode_width)
.await?;
stats.record_join_wait(join_wait);
Ok(stats)
}
/// Pump UDP symbol datagrams into the decoders until a control frame arrives.
///
/// The sender finishes a spray round and *then* sends `ObjectComplete` on TCP,
/// so by interleaving `udp.recv` with `control.recv` we absorb the bulk symbols
/// and return as soon as the round's control marker lands. The UDP branch mirrors
/// native QUIC's `recv_batch_from` pump: one readiness-driven receive drains all
/// immediately-ready packets, then full batches get a bounded quiet-drain pass.
async fn pump_until_control<S>(
cx: &Cx,
control: &mut FrameTransport<S>,
udp: &mut RqReceiverUdpFanout,
tag: u64,
auth_required: bool,
symbol_auth: Option<&SecurityContext>,
rbuf: &mut [u8],
decoders: &mut [EntryDecoder],
symbol_size: u16,
symbols_accepted: &mut u64,
round_stats: &mut RqDatagramRoundStats,
trace_intake: bool,
) -> Result<Frame, RqError>
where
S: AsyncReadExt + AsyncWriteExt + Unpin,
{
use std::future::poll_fn;
use std::pin::Pin;
use std::task::Poll;
enum Ready {
Control(usize),
Udp {
socket_index: usize,
batch: crate::net::UdpRecvBatch,
},
}
let packet_size = rbuf.len();
let mut cbuf = vec![0u8; 65536];
let mut pumped: u64 = 0;
loop {
cx.checkpoint().map_err(|_| RqError::Cancelled)?;
round_stats
.record_decode_stats(drain_ready_decodes_if_pending(cx, decoders, symbol_size).await?);
// 1) First, non-blockingly drain whatever the control codec already has
// buffered (a prior read may have pulled the frame in with symbols).
if let Some(frame) = control
.codec
.decode(&mut control.rbuf)
.map_err(|e| RqError::Frame(e.to_string()))?
{
rqtrace!(
"pump: returning {:?} after {pumped} udp datagrams",
frame.frame_type()
);
return Ok(frame);
}
// 2) Poll both the control stream and a readiness-driven UDP fanout batch.
// Whichever is ready makes progress; if only UDP is ready we keep
// pumping symbols. Both register their waker via task_cx, so the task
// parks until EITHER fd is ready — a biased two-way select.
let recv_started = trace_intake.then(Instant::now);
let ready = {
poll_fn(|task_cx| {
// UDP first so bulk data drains promptly under load.
match udp.poll_recv_batch_any(task_cx, RQ_INBOUND_PUMP_BATCH, packet_size) {
Poll::Ready(Ok((socket_index, batch))) => {
return Poll::Ready(Ok::<Ready, std::io::Error>(Ready::Udp {
socket_index,
batch,
}));
}
Poll::Ready(Err(e)) => return Poll::Ready(Err(e)),
Poll::Pending => {}
}
let mut read_buf = ReadBuf::new(&mut cbuf);
match Pin::new(&mut control.stream).poll_read(task_cx, &mut read_buf) {
Poll::Ready(Ok(())) => Poll::Ready(Ok(Ready::Control(read_buf.filled().len()))),
Poll::Ready(Err(e)) => Poll::Ready(Err(e)),
Poll::Pending => Poll::Pending,
}
})
.await?
};
let recv_elapsed = recv_started.map(|started| started.elapsed());
match ready {
Ready::Udp {
socket_index,
mut batch,
} => {
let mut received_len = batch.packets.len();
let mut batches = 1usize;
let mut stats = feed_datagram_batch_to_decoders(
cx,
&batch,
tag,
auth_required,
symbol_auth,
decoders,
symbol_size,
trace_intake,
)
.await?;
if let Some(elapsed) = recv_elapsed {
stats.record_recv_elapsed(elapsed);
}
pumped = pumped.saturating_add(stats.observed);
*symbols_accepted = (*symbols_accepted).saturating_add(stats.accepted);
round_stats.merge(stats);
round_stats.record_decode_stats(
drain_ready_decodes_if_pending(cx, decoders, symbol_size).await?,
);
udp.recycle_recv_batch(&mut batch, RQ_INBOUND_PUMP_BATCH);
while received_len == RQ_INBOUND_PUMP_BATCH {
if batches >= RQ_INBOUND_PUMP_MAX_DRAIN_BATCHES {
rqtrace!(
"pump: udp batch drain budget exhausted after {batches} batches and {pumped} accepted datagrams"
);
break;
}
let tail_recv_started = trace_intake.then(Instant::now);
let mut tail = match crate::time::timeout(
cx.now(),
RQ_INBOUND_PUMP_DRAIN_GRACE,
udp.recv_batch_from_socket(
socket_index,
RQ_INBOUND_PUMP_BATCH,
packet_size,
),
)
.await
{
Ok(Ok(batch)) => batch,
Ok(Err(e)) => return Err(RqError::Io(e)),
Err(_elapsed) => break,
};
let tail_recv_elapsed = tail_recv_started.map(|started| started.elapsed());
received_len = tail.packets.len();
if received_len == 0 {
break;
}
let mut stats = feed_datagram_batch_to_decoders(
cx,
&tail,
tag,
auth_required,
symbol_auth,
decoders,
symbol_size,
trace_intake,
)
.await?;
if let Some(elapsed) = tail_recv_elapsed {
stats.record_tail_drain_elapsed(elapsed);
}
pumped = pumped.saturating_add(stats.observed);
*symbols_accepted = (*symbols_accepted).saturating_add(stats.accepted);
round_stats.merge(stats);
round_stats.record_decode_stats(
drain_ready_decodes_if_pending(cx, decoders, symbol_size).await?,
);
udp.recycle_recv_batch(&mut tail, RQ_INBOUND_PUMP_BATCH);
batches = batches.saturating_add(1);
}
}
Ready::Control(n) => {
if n == 0 {
return Err(RqError::Io(std::io::Error::new(
std::io::ErrorKind::UnexpectedEof,
"control stream closed mid-transfer",
)));
}
control.rbuf.extend_from_slice(&cbuf[..n]);
if let Some(frame) = control
.codec
.decode(&mut control.rbuf)
.map_err(|e| RqError::Frame(e.to_string()))?
{
return Ok(frame);
}
}
}
}
}
/// Drain UDP symbols that raced behind the TCP round marker.
///
/// `ObjectComplete` only proves the sender has finished a spray round; it does
/// not prove the receiver has drained every datagram already queued locally. The
/// drain stops after a quiet window with no matching ATP-RQ symbol, with a hard
/// cap of 8x that window so stale or hostile UDP traffic cannot pin the task.
async fn drain_round_tail(
cx: &Cx,
udp: &mut RqReceiverUdpFanout,
tag: u64,
auth_required: bool,
symbol_auth: Option<&SecurityContext>,
rbuf: &mut [u8],
quiet_window: Duration,
decoders: &mut [EntryDecoder],
symbol_size: u16,
symbols_accepted: &mut u64,
round_stats: &mut RqDatagramRoundStats,
trace_intake: bool,
) -> Result<u64, RqError> {
if quiet_window.is_zero() {
return Ok(0);
}
use std::future::poll_fn;
use std::pin::Pin;
use std::task::Poll;
let mut quiet_sleep = crate::time::Sleep::after(cx.now_for_observability(), quiet_window);
let hard_cap = quiet_window.saturating_mul(8).max(Duration::from_millis(1));
let mut hard_sleep = crate::time::Sleep::after(cx.now_for_observability(), hard_cap);
let mut drained = 0u64;
loop {
cx.checkpoint().map_err(|_| RqError::Cancelled)?;
let drain_started = trace_intake.then(Instant::now);
let ready = poll_fn(|task_cx| {
if Pin::new(&mut hard_sleep).poll(task_cx).is_ready() {
return Poll::Ready(Ok::<Option<usize>, std::io::Error>(None));
}
match udp.poll_recv_any(task_cx, rbuf) {
Poll::Ready(Ok((_socket_index, n))) => return Poll::Ready(Ok(Some(n))),
Poll::Ready(Err(e)) => return Poll::Ready(Err(e)),
Poll::Pending => {}
}
if Pin::new(&mut quiet_sleep).poll(task_cx).is_ready() {
return Poll::Ready(Ok(None));
}
Poll::Pending
})
.await?;
let drain_elapsed = drain_started.map(|started| started.elapsed());
let Some(n) = ready else {
return Ok(drained);
};
let ingest = feed_datagram_to_decoders(
cx,
rbuf,
n,
tag,
auth_required,
symbol_auth,
decoders,
symbol_size,
trace_intake,
)
.await?;
if ingest.observed {
drained += 1;
round_stats.record(ingest);
if let Some(elapsed) = drain_elapsed {
round_stats.record_tail_drain_elapsed(elapsed);
}
if ingest.accepted {
*symbols_accepted = (*symbols_accepted).saturating_add(1);
}
quiet_sleep.reset_after(cx.now_for_observability(), quiet_window);
}
round_stats
.record_decode_stats(drain_ready_decodes_if_pending(cx, decoders, symbol_size).await?);
if drained > 0 && drained % 512 == 0 {
crate::runtime::yield_now().await;
}
}
}
/// Run a persistent accept loop, handling each control connection as one
/// receive.
///
/// Returns when the capability context is cancelled. Connection-level errors are
/// reported via `on_result` and do not stop the loop.
pub async fn serve<F>(
cx: &Cx,
control_listener: TcpListener,
udp_bind_ip: String,
dest_dir: PathBuf,
config: RqConfig,
peer_id: String,
mut on_result: F,
) -> Result<(), RqError>
where
F: FnMut(Result<ReceiveReport, RqError>),
{
loop {
if cx.is_cancel_requested() {
return Ok(());
}
let (stream, peer) = control_listener.accept().await?;
let result = receive_connection(
cx,
stream,
peer,
&udp_bind_ip,
&dest_dir,
config.clone(),
&peer_id,
)
.await;
on_result(result);
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn hello_ack_udp_ports_defaults_to_legacy_port() {
let ack = HelloAck {
accepted: true,
peer_id: "receiver".to_string(),
udp_port: 8472,
udp_ports: Vec::new(),
control_source_stream: false,
reason: None,
};
assert_eq!(hello_ack_udp_ports(&ack).as_slice(), &[8472]);
}
#[test]
fn hello_ack_udp_ports_drive_socket_round_robin() {
let ack = HelloAck {
accepted: true,
peer_id: "receiver".to_string(),
udp_port: 3001,
udp_ports: vec![3001, 3002, 3003],
control_source_stream: false,
reason: None,
};
let ports = hello_ack_udp_ports(&ack);
let peer: SocketAddr = "192.0.2.10:8472".parse().unwrap();
let mapped_ports = (0..7)
.map(|socket_index| {
receiver_udp_addr_for_socket(peer, &ports, socket_index)
.unwrap()
.port()
})
.collect::<Vec<_>>();
assert_eq!(mapped_ports, vec![3001, 3002, 3003, 3001, 3002, 3003, 3001]);
}
#[test]
fn parallel_decode_spawn_gate_respects_matrix5_width_cap() {
assert!(
RQ_MAX_PENDING_DECODE_JOBS_PER_ENTRY <= RQ_MAX_PENDING_DECODE_JOBS_PER_TRANSFER_HARD,
"entry decode width must not exceed the transfer hard cap"
);
assert!(
RQ_MAX_PENDING_DECODE_JOBS_PER_TRANSFER_HARD >= 64,
"large-object repair decode must fan out on 64-core receivers"
);
assert!(can_spawn_parallel_decode(
0,
RQ_MAX_PENDING_DECODE_JOBS_PER_ENTRY
));
assert!(can_spawn_parallel_decode(
RQ_MAX_PENDING_DECODE_JOBS_PER_ENTRY - 1,
RQ_MAX_PENDING_DECODE_JOBS_PER_ENTRY
));
assert!(!can_spawn_parallel_decode(
RQ_MAX_PENDING_DECODE_JOBS_PER_ENTRY,
RQ_MAX_PENDING_DECODE_JOBS_PER_ENTRY
));
assert!(
!can_spawn_parallel_decode(0, 0),
"zero decode width must not spawn"
);
assert!(
!can_spawn_parallel_decode(0, 1),
"one-wide inline decode gate must not spawn"
);
}
#[test]
fn decode_round_stats_merge_preserves_trace_counters() {
let mut first = RqDecodeRoundStats::default();
first.record_attempt(BlockDecodeKind::RaptorQRepair, Duration::from_micros(7));
first.record_join_wait(Duration::from_micros(11));
first.record_queued_job(3);
first.record_inline_job();
first.record_spawn_denial();
first.record_entry_cap_saturation();
let mut second = RqDecodeRoundStats::default();
second.record_attempt(BlockDecodeKind::SourceComplete, Duration::from_micros(13));
second.record_join_wait(Duration::from_micros(17));
second.record_queued_job(9);
second.record_transfer_cap_saturation();
second.apply_micros = 19;
second.persist_micros = 23;
first.merge(second);
assert_eq!(first.attempts, 2);
assert_eq!(first.repair_attempts, 1);
assert_eq!(first.source_complete_attempts, 1);
assert_eq!(first.decode_micros, 20);
assert_eq!(first.join_wait_micros, 28);
assert_eq!(first.queued_jobs, 2);
assert_eq!(first.inline_jobs, 1);
assert_eq!(first.spawn_denials, 1);
assert_eq!(first.entry_cap_saturations, 1);
assert_eq!(first.transfer_cap_saturations, 1);
assert_eq!(first.pending_peak, 9);
assert_eq!(first.apply_micros, 19);
assert_eq!(first.persist_micros, 23);
}
#[test]
fn decode_core_limit_keeps_parallelism_on_four_core_hosts() {
assert_eq!(rq_decode_core_limit_for_available(1), 1);
assert_eq!(rq_decode_core_limit_for_available(2), 1);
assert_eq!(rq_decode_core_limit_for_available(4), 3);
assert_eq!(rq_decode_core_limit_for_available(8), 6);
assert_eq!(rq_decode_core_limit_for_available(16), 12);
assert_eq!(rq_decode_core_limit_for_available(64), 60);
assert_eq!(
rq_decode_core_limit_for_available(96),
RQ_MAX_PENDING_DECODE_JOBS_PER_TRANSFER_HARD
);
}
#[test]
fn decode_width_uses_receiver_blocking_pool_capacity() {
let config = RqConfig::default();
let size_500m = 500 * 1024 * 1024;
let block_500m = effective_max_block_size_for_largest_entry(&config, size_500m)
.expect("500M fixture must fit default RQ geometry");
let dec_500m = decode_width_fixture_entry(size_500m as u64, block_500m, config.symbol_size);
let pool = crate::runtime::blocking_pool::BlockingPool::new(4, 4);
let cx = Cx::new(
crate::types::RegionId::new_for_test(31, 1),
crate::types::TaskId::new_for_test(31, 0),
crate::types::Budget::INFINITE,
)
.with_blocking_pool_handle(Some(pool.handle()));
let budget = rq_decode_width_budget_snapshot_for_cx(
&cx,
std::slice::from_ref(&dec_500m),
config.symbol_size,
);
assert_eq!(
budget.core_limit, 3,
"four blocking threads should reserve one core for UDP/control and leave three decode slots"
);
assert_eq!(
budget.effective,
budget.core_limit.min(budget.memory_limit),
"500M geometry should use the receiver blocking-pool width instead of collapsing to host available_parallelism"
);
}
#[test]
fn auth_verify_width_uses_receiver_blocking_pool_without_decode_size_gate() {
let no_pool_cx = Cx::for_testing();
assert_eq!(
rq_auth_verify_width_for_cx(&no_pool_cx, RQ_AUTH_VERIFY_PARALLEL_MIN_SYMBOLS),
1,
"lab/no-pool contexts must verify inline instead of trying to spawn"
);
let pool = crate::runtime::blocking_pool::BlockingPool::new(4, 4);
let cx = Cx::new(
crate::types::RegionId::new_for_test(41, 1),
crate::types::TaskId::new_for_test(41, 0),
crate::types::Budget::INFINITE,
)
.with_blocking_pool_handle(Some(pool.handle()));
assert_eq!(
rq_auth_verify_width_for_cx(&cx, 512),
3,
"auth verification must use blocking-pool CPU width even for 50M-class transfers where repair decode is size-gated"
);
assert_eq!(
rq_auth_verify_width_for_cx(&cx, RQ_AUTH_VERIFY_PARALLEL_MIN_SYMBOLS - 1),
1,
"tiny batches should stay inline"
);
}
#[test]
fn decode_width_budget_reports_effective_core_and_memory_caps() {
let budget = rq_decode_width_budget_snapshot(&[], DEFAULT_SYMBOL_SIZE);
assert!(budget.effective >= 1);
assert!(budget.core_limit >= budget.effective);
assert!(budget.memory_limit >= budget.effective);
assert_eq!(budget.max_block_size, DEFAULT_MAX_BLOCK_SIZE);
assert_eq!(
budget.job_memory_bytes,
rq_decode_job_memory_estimate_bytes(DEFAULT_MAX_BLOCK_SIZE, DEFAULT_SYMBOL_SIZE)
);
assert_eq!(budget.effective, budget.core_limit.min(budget.memory_limit));
}
#[test]
fn decode_entry_width_gate_keeps_50m_geometry_sequential() {
let config = RqConfig::default();
let workload_50m = 50 * 1024 * 1024;
let max_block_size = effective_max_block_size_for_largest_entry(&config, workload_50m)
.expect("50M must fit default RQ transfer geometry");
let block_count =
entry_source_block_count_for_geometry(workload_50m as u64, max_block_size, 0);
assert!(
block_count < RQ_PARALLEL_DECODE_MIN_SOURCE_BLOCKS,
"fixture should represent the 50M small-object cell: block_count={block_count} threshold={RQ_PARALLEL_DECODE_MIN_SOURCE_BLOCKS}"
);
assert!(!should_parallel_decode_entry_geometry(
workload_50m as u64,
max_block_size,
0
));
assert_eq!(
entry_decode_width_budget_for_geometry(workload_50m as u64, max_block_size, 0, 64),
0,
"50M geometry should close the parallel decode budget"
);
}
#[test]
fn decode_entry_width_gate_rejects_small_high_block_count_geometry() {
let config = RqConfig::default();
let workload_50m = 50 * 1024 * 1024;
let observed_blocks = RQ_PARALLEL_DECODE_MIN_SOURCE_BLOCKS;
assert!(!should_parallel_decode_entry_geometry(
workload_50m as u64,
usize::from(config.symbol_size).saturating_mul(TARGET_SOURCE_SYMBOLS_PER_BLOCK),
observed_blocks
));
assert_eq!(
entry_decode_width_budget_for_geometry(
workload_50m as u64,
usize::from(config.symbol_size).saturating_mul(TARGET_SOURCE_SYMBOLS_PER_BLOCK),
observed_blocks,
RQ_MAX_PENDING_DECODE_JOBS_PER_TRANSFER_HARD,
),
0,
"50M-class entries must stay sequential even when block count reaches the old fanout threshold"
);
}
#[test]
fn decode_memory_budget_keeps_500m_geometry_wide() {
let config = RqConfig::default();
let workload_500m = 500 * 1024 * 1024;
let max_block_size = effective_max_block_size_for_largest_entry(&config, workload_500m)
.expect("500M must fit default RQ transfer geometry");
let block_count =
entry_source_block_count_for_geometry(workload_500m as u64, max_block_size, 0);
let job_memory_bytes =
rq_decode_job_memory_estimate_bytes(max_block_size, config.symbol_size);
let memory_limit = RQ_DECODE_JOB_MEMORY_BUDGET_BYTES / job_memory_bytes;
let effective_memory_width = memory_limit.min(RQ_MAX_PENDING_DECODE_JOBS_PER_TRANSFER_HARD);
assert!(
effective_memory_width >= 48,
"500M decode should not collapse back to narrow fan-out: max_block_size={max_block_size} job_memory_bytes={job_memory_bytes} memory_limit={memory_limit} effective_memory_width={effective_memory_width}"
);
assert!(
effective_memory_width <= RQ_MAX_PENDING_DECODE_JOBS_PER_TRANSFER_HARD,
"effective memory gate should stay within the hard transfer cap"
);
assert!(
should_parallel_decode_entry_geometry(workload_500m as u64, max_block_size, 0),
"500M geometry should remain eligible for parallel decode: block_count={block_count} threshold={RQ_PARALLEL_DECODE_MIN_SOURCE_BLOCKS}"
);
assert_eq!(
entry_decode_width_budget_for_geometry(
workload_500m as u64,
max_block_size,
0,
effective_memory_width
),
block_count
.min(RQ_MAX_PENDING_DECODE_JOBS_PER_ENTRY)
.min(effective_memory_width)
);
}
fn decode_width_fixture_entry(
size: u64,
max_block_size: usize,
symbol_size: u16,
) -> EntryDecoder {
EntryDecoder {
index: 0,
object_id: ObjectId::new(0xD3C0_D3C0, 0),
size,
pipeline: None,
complete: false,
staging_path: PathBuf::new(),
staging_write_offset: 0,
staging_file_len: size,
staging_shared: false,
staging_created: false,
staging_file: None,
staging_cursor: None,
staging_unflushed_bytes: 0,
cache_staging_file: false,
bytes_written: 0,
max_block_size,
source_streaming: true,
source_blocks: source_block_progress_for(size, max_block_size, symbol_size)
.expect("fixture must fit source-block table"),
pending_decodes: Vec::new(),
inc: None,
inc_digest: None,
source_write_buffer: Vec::new(),
source_write_buffer_offset: None,
}
}
#[test]
fn decode_width_size_gate_keeps_50m_sequential_and_500m_parallel() {
let config = RqConfig::default();
let size_50m = 50 * 1024 * 1024;
let block_50m = effective_max_block_size_for_largest_entry(&config, size_50m)
.expect("50M fixture must fit default RQ geometry");
let dec_50m = decode_width_fixture_entry(size_50m as u64, block_50m, config.symbol_size);
assert!(
!should_parallel_decode_entry(&dec_50m),
"50M decode should stay sequential: block_count={} max_block_size={block_50m}",
entry_source_block_count(&dec_50m)
);
assert_eq!(
entry_decode_width_budget(&dec_50m, RQ_MAX_PENDING_DECODE_JOBS_PER_TRANSFER_HARD),
0,
"50M repair decode fanout should be fully disabled"
);
assert_eq!(
rq_decode_width_budget_snapshot(std::slice::from_ref(&dec_50m), config.symbol_size)
.effective,
0,
"all-small transfers should close the transfer-wide parallel decode budget"
);
let size_500m = 500 * 1024 * 1024;
let block_500m = effective_max_block_size_for_largest_entry(&config, size_500m)
.expect("500M fixture must fit default RQ geometry");
let dec_500m = decode_width_fixture_entry(size_500m as u64, block_500m, config.symbol_size);
assert!(
should_parallel_decode_entry(&dec_500m),
"500M decode should retain parallel fanout: block_count={} max_block_size={block_500m}",
entry_source_block_count(&dec_500m)
);
assert!(
entry_decode_width_budget(&dec_500m, RQ_MAX_PENDING_DECODE_JOBS_PER_TRANSFER_HARD)
>= 48,
"500M repair decode should keep wide fanout after size gating"
);
let budget_500m =
rq_decode_width_budget_snapshot(std::slice::from_ref(&dec_500m), config.symbol_size);
assert_eq!(
budget_500m.effective,
budget_500m.core_limit.min(budget_500m.memory_limit).max(1),
"500M transfer decode budget should not be narrowed by the size gate"
);
}
#[test]
fn rq_pacing_carries_path_rate_for_congestion_controller() {
let pacing = RqSprayPacing::from_rate(
RQ_COLD_START_PACING_BPS,
1024,
RQ_COLD_START_BURST_SYMBOLS,
None,
false,
);
let symbol_bytes =
1024_u64.saturating_add(u64::try_from(AUTH_DGRAM_HEADER).unwrap_or(u64::MAX));
assert_eq!(
pacing.path_rate_bps,
RQ_COLD_START_PACING_BPS.saturating_mul(8)
);
assert_eq!(pacing.datagram_bytes, u32::try_from(symbol_bytes).unwrap());
assert_eq!(
pacing.max_burst_size,
u32::try_from(RQ_COLD_START_BURST_SYMBOLS).unwrap()
);
}
#[test]
fn rq_round0_clean_ramp_requires_loss_free_target() {
let clean = RqConfig {
symbol_size: 1200,
repair_overhead: 1.0,
round0_loss_target: 0.0,
debug_drop_one_in: 0,
..RqConfig::default()
};
let pacing = RqSprayPacing::cold_start(clean.symbol_size);
assert!(round0_clean_ramp_enabled(&clean, pacing));
assert!(round0_clean_ramp_enabled(
&RqConfig {
repair_overhead: RQ_SMALL_CLEAN_SOURCE_ONLY_MAX_REPAIR_OVERHEAD,
..clean.clone()
},
RqSprayPacing::from_rate(
RQ_COLD_START_PACING_BPS / 4,
clean.symbol_size,
RQ_ADAPTIVE_BURST_SYMBOLS,
None,
false,
)
));
for blocked in [
RqConfig {
round0_loss_target: RQ_ROUND0_TARGET_LOSS_ENABLE_MIN / 5.0,
..clean.clone()
},
RqConfig {
round0_loss_target: RQ_ROUND0_TARGET_LOSS_ENABLE_MIN,
..clean.clone()
},
RqConfig {
round0_loss_target: 0.02,
..clean.clone()
},
RqConfig {
round0_loss_target: 0.10,
..clean.clone()
},
RqConfig {
repair_overhead: 1.01,
..clean.clone()
},
RqConfig {
debug_drop_one_in: 7,
..clean.clone()
},
] {
assert!(
!round0_clean_ramp_enabled(&blocked, pacing),
"clean ramp must stay off for good/lossy/debug/repair-configured round 0"
);
}
let fanout = RqConfig {
udp_fanout: 8,
..clean.clone()
};
assert!(
round0_clean_ramp_enabled(&fanout, pacing),
"fanout should share the aggregate clean ramp instead of disabling it"
);
}
#[test]
fn rq_round0_clean_ramp_additively_probes_inside_source_round() {
let mut pacing = RqSprayPacing::cold_start(1200);
let mut ramp = RqRound0CleanPacingRamp::new(RQ_ROUND0_CLEAN_RAMP_MAX_PACING_BPS);
let step_datagrams =
RQ_ROUND0_CLEAN_RAMP_STEP_BYTES.div_ceil(u64::from(pacing.datagram_bytes));
ramp.sent_datagrams = step_datagrams.saturating_sub(1);
let first = ramp
.observe_datagram(&mut pacing)
.expect("first clean-ramp step");
assert_eq!(
first.new_rate_bytes_per_sec,
RQ_COLD_START_PACING_BPS + RQ_ROUND0_CLEAN_RAMP_ADD_BYTES_PER_S
);
assert_eq!(
pacing.rate_bytes_per_sec(),
RQ_COLD_START_PACING_BPS + RQ_ROUND0_CLEAN_RAMP_ADD_BYTES_PER_S
);
ramp.sent_datagrams = ramp
.next_step_bytes
.div_ceil(u64::from(pacing.datagram_bytes))
.saturating_sub(1);
let second = ramp
.observe_datagram(&mut pacing)
.expect("second clean-ramp step");
assert_eq!(
second.new_rate_bytes_per_sec,
RQ_COLD_START_PACING_BPS + RQ_ROUND0_CLEAN_RAMP_ADD_BYTES_PER_S * 2
);
assert_eq!(
pacing.rate_bytes_per_sec(),
RQ_COLD_START_PACING_BPS + RQ_ROUND0_CLEAN_RAMP_ADD_BYTES_PER_S * 2
);
while pacing.rate_bytes_per_sec() < RQ_ROUND0_CLEAN_RAMP_MAX_PACING_BPS {
ramp.sent_datagrams = ramp
.next_step_bytes
.div_ceil(u64::from(pacing.datagram_bytes))
.saturating_sub(1);
let _ = ramp
.observe_datagram(&mut pacing)
.expect("clean ramp should keep stepping until max");
}
assert_eq!(
pacing.rate_bytes_per_sec(),
RQ_ROUND0_CLEAN_RAMP_MAX_PACING_BPS
);
assert!(
pacing.rate_bytes_per_sec() > RQ_MAX_PACING_BPS,
"clean round-0 probe must be able to test beyond the adaptive cap"
);
}
#[test]
fn rq_round0_clean_ramp_caps_fanout_aggregate_rate() {
let single = RqConfig {
udp_fanout: 1,
..RqConfig::default()
};
let fanout = RqConfig {
udp_fanout: 8,
..RqConfig::default()
};
assert_eq!(
round0_clean_ramp_max_rate(&single),
RQ_ROUND0_CLEAN_RAMP_MAX_PACING_BPS
);
assert_eq!(
round0_clean_ramp_max_rate(&fanout),
RQ_ROUND0_CLEAN_RAMP_FANOUT_MAX_PACING_BPS
);
let mut pacing = RqSprayPacing::cold_start(1200);
let mut ramp = RqRound0CleanPacingRamp::new(round0_clean_ramp_max_rate(&fanout));
while pacing.rate_bytes_per_sec() < RQ_ROUND0_CLEAN_RAMP_FANOUT_MAX_PACING_BPS {
ramp.sent_datagrams = ramp
.next_step_bytes
.div_ceil(u64::from(pacing.datagram_bytes))
.saturating_sub(1);
let report = ramp
.observe_datagram(&mut pacing)
.expect("fanout ramp should step until aggregate cap");
assert_eq!(
report.max_rate_bytes_per_sec,
RQ_ROUND0_CLEAN_RAMP_FANOUT_MAX_PACING_BPS
);
}
assert_eq!(
pacing.rate_bytes_per_sec(),
RQ_ROUND0_CLEAN_RAMP_FANOUT_MAX_PACING_BPS
);
}
#[test]
fn rq_round0_clean_ramp_resets_on_feedback_pacer_reconfigure() {
let config = RqConfig {
symbol_size: 1200,
repair_overhead: 1.0,
round0_loss_target: 0.0,
..RqConfig::default()
};
let mut pacer = RqSprayPacer::new_round0(RqSprayPacing::cold_start(1200), &config, false);
assert!(pacer.round0_ramp.is_some());
pacer.configure_with_shared_decision(
RqSprayPacing::from_rate(
RQ_COLD_START_PACING_BPS / 2,
1200,
RQ_ADAPTIVE_BURST_SYMBOLS,
Some(Duration::from_millis(200)),
true,
),
None,
);
assert!(pacer.round0_ramp.is_none());
assert_eq!(
pacer.pacing().rate_bytes_per_sec(),
RQ_COLD_START_PACING_BPS / 2
);
}
#[test]
fn rq_round0_clean_pacer_reports_ramped_rate_to_window_probe() {
let config = RqConfig {
symbol_size: 1200,
repair_overhead: 1.0,
round0_loss_target: 0.0,
..RqConfig::default()
};
let mut pacer = RqSprayPacer::new_round0(RqSprayPacing::cold_start(1200), &config, false);
let datagrams = RQ_ROUND0_CLEAN_RAMP_STEP_BYTES
.div_ceil(u64::from(pacer.pacing().datagram_bytes.max(1)));
for _ in 0..datagrams {
pacer.observe_datagram_sent();
}
let probe = RqSenderWindowProbe::new(
pacer.pacing(),
1,
config.symbol_size,
Duration::from_millis(1),
Duration::from_millis(200),
);
assert_eq!(
probe.configured_rate_bytes_per_sec,
RQ_COLD_START_PACING_BPS + RQ_ROUND0_CLEAN_RAMP_ADD_BYTES_PER_S,
"ATP_RQ_TRACE window_probe should expose the within-round clean ramp"
);
}
fn rq_test_path_estimate(config: &RqConfig, bytes_per_second: f64) -> PathEstimate {
PathEstimate {
rtt_s: 0.050,
loss_p_hat: 0.0,
loss_p_bar: 0.0,
bw_median_bps: bytes_per_second,
bw_trough_bps: bytes_per_second,
enc_symbols_per_s: RQ_ASSUMED_DECODE_SYMBOLS_PER_S,
dec_symbols_per_s: RQ_ASSUMED_DECODE_SYMBOLS_PER_S,
coding_ref_k: fixed_block_k(config),
coding_gamma: RQ_CODING_GAMMA,
samples: 1,
}
}
fn rq_test_block_plan(config: &RqConfig) -> BlockPlan {
BlockPlan {
k: fixed_block_k(config),
overhead: 0.0,
fanout: 1,
}
}
#[test]
fn rq_pacing_preserves_measured_slow_link_without_loss() {
let config = RqConfig::default();
let mut state = RqAdaptiveSendState::new(7, &config, 1);
let measured = 2_u64 * 1024 * 1024;
state.est = rq_test_path_estimate(&config, measured as f64);
let rate = state.pacing_rate_for(rq_test_block_plan(&config), &config);
assert_eq!(rate, measured);
}
#[test]
fn rq_pacing_floors_mild_loss_collapse() {
let config = RqConfig::default();
let mut state = RqAdaptiveSendState::new(7, &config, 1);
let collapsed = 42_u64 * 1024;
state.est = rq_test_path_estimate(&config, collapsed as f64);
state.loss_ema = 0.001;
state.loss_bar = 0.01;
state.pacing_loss_ema = 0.001;
let rate = state.pacing_rate_for(rq_test_block_plan(&config), &config);
assert!(
rate >= RQ_COLD_START_PACING_BPS / 2,
"mild loss should not pace a repair round at {rate} B/s"
);
assert!(
rate > collapsed.saturating_mul(100),
"floor should break the self-reinforcing 42KB/s collapse"
);
}
#[test]
fn rq_pacing_floor_stays_off_for_regime_shift_loss() {
let config = RqConfig::default();
let mut state = RqAdaptiveSendState::new(7, &config, 1);
let collapsed = 42_u64 * 1024;
state.est = rq_test_path_estimate(&config, collapsed as f64);
state.pacing_loss_ema = RQ_MILD_LOSS_PACING_MAX_LOSS * 2.0;
state.loss_bar = RQ_REGIME_SHIFT_LOSS_DELTA;
let rate = state.pacing_rate_for(rq_test_block_plan(&config), &config);
assert_eq!(rate, RQ_MIN_PACING_BPS);
}
#[test]
fn rq_pacing_mild_loss_floor_overrides_stale_low_cap() {
let config = RqConfig::default();
let mut state = RqAdaptiveSendState::new(7, &config, 1);
state.est = rq_test_path_estimate(&config, 32.0 * 1024.0 * 1024.0);
state.controller.update_estimate(state.est);
state.loss_ema = 0.01;
state.loss_bar = 0.02;
state.pacing_loss_ema = RQ_MILD_LOSS_PACING_MAX_LOSS / 2.0;
state.loss_pacing_cap_bps = Some(RQ_COLD_START_PACING_BPS / 4);
let tuning = state.round_tuning(&config);
assert_eq!(
tuning.pacing.path_rate_bps,
state.mild_loss_pacing_floor_bps(&config).saturating_mul(8),
"stale mild-loss caps must not reintroduce the pacing crawl"
);
}
#[test]
fn rq_good_link_source_first_feedback_uses_full_cold_start_floor() {
let config = RqConfig {
round0_loss_target: RQ_ROUND0_TARGET_LOSS_ENABLE_MIN / 5.0,
..RqConfig::default()
};
let mut state = RqAdaptiveSendState::new(7, &config, 1);
state.est = rq_test_path_estimate(&config, 32.0 * 1024.0 * 1024.0);
state.controller.update_estimate(state.est);
state.loss_ema = 0.01;
state.loss_bar = RQ_PENDING_PRESSURE_LOSS_FLOOR;
state.pacing_loss_ema = config.round0_loss_target;
state.loss_pacing_cap_bps = Some(RQ_COLD_START_PACING_BPS / 4);
let tuning = state.round_tuning(&config);
assert_eq!(
tuning.pacing.rate_bytes_per_sec(),
RQ_COLD_START_PACING_BPS,
"sub-threshold good-link feedback should recover at cold-start instead of the half-rate repair floor"
);
}
#[test]
fn rq_bad_link_feedback_keeps_conservative_mild_loss_floor() {
let config = RqConfig {
round0_loss_target: 0.02,
..RqConfig::default()
};
let mut state = RqAdaptiveSendState::new(7, &config, 1);
state.est = rq_test_path_estimate(&config, 32.0 * 1024.0 * 1024.0);
state.controller.update_estimate(state.est);
state.loss_ema = 0.01;
state.loss_bar = RQ_PENDING_PRESSURE_LOSS_FLOOR;
state.pacing_loss_ema = config.round0_loss_target;
state.loss_pacing_cap_bps = Some(RQ_COLD_START_PACING_BPS / 4);
let tuning = state.round_tuning(&config);
assert_eq!(
tuning.pacing.rate_bytes_per_sec(),
RQ_BAD_LINK_ROUND0_PACING_BPS,
"configured bad-link repair-target cells must pace near the 50 mbit pipe instead of inheriting the good-link floor"
);
}
#[test]
fn rq_aimd_halves_rate_on_receiver_observed_loss() {
let config = RqConfig {
symbol_size: 1200,
..RqConfig::default()
};
let mut state = RqAdaptiveSendState::new(7, &config, 1);
let total_bytes = 50_u64 * 1024 * 1024;
let digests = [EntryDigest {
rel_path: "large.bin".to_string(),
size: total_bytes,
content_id: crate::atp::object::ObjectId::content(
crate::atp::object::ContentId::from_bytes(b"large.bin"),
),
content_sha256: [0; 32],
}];
let pending = BTreeSet::from([0_u32]);
let sent_symbols = 10_000_u64;
state.observe_need_more(
&config,
&digests,
&pending,
sent_symbols,
9_400,
None,
RqDeliverySampleKind::InitialOrRepair,
Duration::from_millis(800),
Duration::from_millis(100),
total_bytes,
);
assert!(state.last_round_loss_fraction > aimd_loss_decrease_threshold(&config));
assert_eq!(state.aimd_rate_bps, RQ_COLD_START_PACING_BPS / 2);
}
#[test]
fn rq_aimd_prefers_explicit_receiver_loss_fraction() {
let config = RqConfig {
symbol_size: 1200,
..RqConfig::default()
};
let mut state = RqAdaptiveSendState::new(7, &config, 1);
let total_bytes = 50_u64 * 1024 * 1024;
let digests = [EntryDigest {
rel_path: "large.bin".to_string(),
size: total_bytes,
content_id: crate::atp::object::ObjectId::content(
crate::atp::object::ContentId::from_bytes(b"large.bin"),
),
content_sha256: [0; 32],
}];
let pending = BTreeSet::from([0_u32]);
state.observe_need_more(
&config,
&digests,
&pending,
10_000,
10_000,
Some(0.06),
RqDeliverySampleKind::InitialOrRepair,
Duration::from_millis(800),
Duration::from_millis(100),
total_bytes,
);
assert_eq!(state.last_round_loss_fraction, 0.06);
assert_eq!(state.aimd_rate_bps, RQ_COLD_START_PACING_BPS / 2);
}
#[test]
fn rq_aimd_holds_rate_under_configured_loss_target() {
let config = RqConfig {
symbol_size: 1200,
round0_loss_target: 0.10,
..RqConfig::default()
};
let mut state = RqAdaptiveSendState::new(7, &config, 1);
let total_bytes = 50_u64 * 1024 * 1024;
let digests = [EntryDigest {
rel_path: "broken.bin".to_string(),
size: total_bytes,
content_id: crate::atp::object::ObjectId::content(
crate::atp::object::ContentId::from_bytes(b"broken.bin"),
),
content_sha256: [0; 32],
}];
let pending = BTreeSet::from([0_u32]);
state.observe_need_more(
&config,
&digests,
&pending,
10_000,
9_000,
None,
RqDeliverySampleKind::InitialOrRepair,
Duration::from_millis(800),
Duration::from_millis(100),
total_bytes,
);
assert!((state.last_round_loss_fraction - 0.10).abs() < f64::EPSILON * 8.0);
assert_eq!(
state.aimd_rate_bps, RQ_BROKEN_LINK_ROUND0_PACING_BPS,
"expected link loss should neither decrease nor increase the AIMD rate"
);
assert_eq!(
state.loss_pacing_cap_bps, None,
"expected-regime loss must not lower the loss-detector pacing cap (MATRIX-207)"
);
}
#[test]
fn rq_aimd_holds_broken_link_cap_on_zero_loss_feedback() {
let config = RqConfig {
symbol_size: 1200,
round0_loss_target: 0.10,
..RqConfig::default()
};
let mut state = RqAdaptiveSendState::new(7, &config, 1);
let total_bytes = 50_u64 * 1024 * 1024;
let digests = [EntryDigest {
rel_path: "broken.bin".to_string(),
size: total_bytes,
content_id: crate::atp::object::ObjectId::content(
crate::atp::object::ContentId::from_bytes(b"broken-zero-loss"),
),
content_sha256: [0; 32],
}];
let pending = BTreeSet::from([0_u32]);
state.observe_need_more(
&config,
&digests,
&pending,
10_000,
10_000,
Some(0.0),
RqDeliverySampleKind::InitialOrRepair,
Duration::from_millis(800),
Duration::from_millis(200),
total_bytes,
);
assert_eq!(state.last_round_loss_fraction, 0.0);
assert_eq!(
state.aimd_rate_bps, RQ_BROKEN_LINK_ROUND0_PACING_BPS,
"explicitly lossy/broken cells must not treat zero-loss feedback as a clean-link additive increase"
);
assert_eq!(
state.round_tuning(&config).pacing.rate_bytes_per_sec(),
RQ_BROKEN_LINK_ROUND0_PACING_BPS,
"repair rounds must keep the 10 mbit-class broken-link cap until real delivery evidence changes it"
);
}
#[test]
fn rq_aimd_backs_off_when_broken_rank_progress_stalls_despite_zero_loss() {
let config = RqConfig {
symbol_size: 1200,
round0_loss_target: 0.10,
..RqConfig::default()
};
let mut state = RqAdaptiveSendState::new(7, &config, 1);
state.aimd_rate_bps = RQ_COLD_START_PACING_BPS;
let total_bytes = 50_u64 * 1024 * 1024;
let digests = [EntryDigest {
rel_path: "broken.bin".to_string(),
size: total_bytes,
content_id: crate::atp::object::ObjectId::content(
crate::atp::object::ContentId::from_bytes(b"broken-progress-stall"),
),
content_sha256: [0; 32],
}];
let pending = BTreeSet::from([0_u32]);
state.observe_need_more_with_progress(
&config,
&digests,
&pending,
total_bytes,
RqNeedMoreProgress {
pending_rank: Some(100),
pending_rank_columns: Some(43_700),
pending_rank_deficit: Some(43_600),
pending_decode_jobs: Some(0),
},
10_000,
6_000,
Some(0.0),
RqDeliverySampleKind::InitialOrRepair,
Duration::from_millis(800),
Duration::from_millis(200),
total_bytes,
);
assert!(
state.last_round_loss_fraction > aimd_loss_decrease_threshold(&config),
"arrival-corroborated rank stall must override underreported receiver loss"
);
assert_eq!(
state.aimd_rate_bps, RQ_MIN_PACING_BPS,
"stalled rank progress with depressed arrivals should back off to the sender-side delivery floor"
);
assert!(
state.pacing_loss_ema > RQ_MILD_LOSS_PACING_MAX_LOSS,
"progress-derived congestion must feed the pacing/loss detector path"
);
}
#[test]
fn rq_aimd_holds_rate_when_rank_stalls_but_arrivals_complete() {
let config = RqConfig {
symbol_size: 1200,
round0_loss_target: 0.10,
..RqConfig::default()
};
let mut state = RqAdaptiveSendState::new(7, &config, 1);
state.aimd_rate_bps = RQ_COLD_START_PACING_BPS;
let total_bytes = 50_u64 * 1024 * 1024;
let digests = [EntryDigest {
rel_path: "broken.bin".to_string(),
size: total_bytes,
content_id: crate::atp::object::ObjectId::content(
crate::atp::object::ContentId::from_bytes(b"broken-stall-healthy-arrivals"),
),
content_sha256: [0; 32],
}];
let pending = BTreeSet::from([0_u32]);
state.observe_need_more_with_progress(
&config,
&digests,
&pending,
total_bytes,
RqNeedMoreProgress {
pending_rank: Some(100),
pending_rank_columns: Some(43_700),
pending_rank_deficit: Some(43_600),
pending_decode_jobs: Some(0),
},
10_000,
10_000,
None,
RqDeliverySampleKind::InitialOrRepair,
Duration::from_millis(800),
Duration::from_millis(200),
total_bytes,
);
assert!(
state.last_round_loss_fraction <= aimd_loss_decrease_threshold(&config),
"a decode-side stall with complete arrivals is not congestion"
);
assert_eq!(
state.aimd_rate_bps, RQ_COLD_START_PACING_BPS,
"healthy arrivals must veto the rank-stall congestion proxy: slowing the \
sender cannot un-stall a rank-deficient block (500M/broken collapse, MATRIX-207)"
);
assert!(
state.pacing_loss_ema <= RQ_MILD_LOSS_PACING_MAX_LOSS,
"an uncorroborated stall must not poison the pacing loss detector"
);
}
#[test]
fn rq_aimd_holds_broken_cap_when_rank_progress_matches_offer() {
let config = RqConfig {
symbol_size: 1200,
round0_loss_target: 0.10,
..RqConfig::default()
};
let mut state = RqAdaptiveSendState::new(7, &config, 1);
let total_bytes = 50_u64 * 1024 * 1024;
let digests = [EntryDigest {
rel_path: "broken.bin".to_string(),
size: total_bytes,
content_id: crate::atp::object::ObjectId::content(
crate::atp::object::ContentId::from_bytes(b"broken-progress-healthy"),
),
content_sha256: [0; 32],
}];
let pending = BTreeSet::from([0_u32]);
state.observe_need_more_with_progress(
&config,
&digests,
&pending,
total_bytes,
RqNeedMoreProgress {
pending_rank: Some(8_500),
pending_rank_columns: Some(43_700),
pending_rank_deficit: Some(35_200),
pending_decode_jobs: Some(0),
},
10_000,
10_000,
Some(0.0),
RqDeliverySampleKind::InitialOrRepair,
Duration::from_millis(1_000),
Duration::from_millis(200),
total_bytes,
);
assert_eq!(state.last_round_loss_fraction, 0.0);
assert_eq!(
state.aimd_rate_bps, RQ_BROKEN_LINK_ROUND0_PACING_BPS,
"healthy rank progress should not back off below the configured broken-link cap"
);
}
#[test]
fn rq_aimd_recovers_broken_link_cap_after_clean_feedback() {
let config = RqConfig {
symbol_size: 1200,
round0_loss_target: 0.10,
..RqConfig::default()
};
let mut state = RqAdaptiveSendState::new(7, &config, 1);
state.aimd_rate_bps = RQ_MIN_PACING_BPS;
let total_bytes = 50_u64 * 1024 * 1024;
let digests = [EntryDigest {
rel_path: "broken.bin".to_string(),
size: total_bytes,
content_id: crate::atp::object::ObjectId::content(
crate::atp::object::ContentId::from_bytes(b"broken-recovery"),
),
content_sha256: [0; 32],
}];
let pending = BTreeSet::from([0_u32]);
state.observe_need_more(
&config,
&digests,
&pending,
10_000,
10_000,
Some(0.0),
RqDeliverySampleKind::InitialOrRepair,
Duration::from_millis(800),
Duration::from_millis(200),
total_bytes,
);
assert_eq!(
state.aimd_rate_bps, RQ_BROKEN_LINK_ROUND0_PACING_BPS,
"a conservative broken-link decrease must recover up to the 10 mbit-class cap on clean feedback"
);
}
#[test]
fn rq_aimd_holds_rate_for_bad_link_loss_target() {
let config = RqConfig {
symbol_size: 1200,
round0_loss_target: 0.02,
..RqConfig::default()
};
let mut state = RqAdaptiveSendState::new(7, &config, 1);
let total_bytes = 50_u64 * 1024 * 1024;
let digests = [EntryDigest {
rel_path: "bad.bin".to_string(),
size: total_bytes,
content_id: crate::atp::object::ObjectId::content(
crate::atp::object::ContentId::from_bytes(b"bad.bin"),
),
content_sha256: [0; 32],
}];
let pending = BTreeSet::from([0_u32]);
state.observe_need_more(
&config,
&digests,
&pending,
10_000,
9_800,
None,
RqDeliverySampleKind::InitialOrRepair,
Duration::from_millis(800),
Duration::from_millis(100),
total_bytes,
);
assert!((state.last_round_loss_fraction - 0.02).abs() < f64::EPSILON * 8.0);
assert_eq!(
state.aimd_rate_bps, RQ_BAD_LINK_ROUND0_PACING_BPS,
"bad-cell target loss should hold the 50 mbit pacing cap, not pace up into self-loss"
);
}
#[test]
fn rq_aimd_loss_target_backoff_uses_receiver_delivery_rate() {
let config = RqConfig {
symbol_size: 1200,
round0_loss_target: 0.02,
..RqConfig::default()
};
let mut state = RqAdaptiveSendState::new(7, &config, 1);
let total_bytes = 500_u64 * 1024 * 1024;
let digests = [EntryDigest {
rel_path: "bad.bin".to_string(),
size: total_bytes,
content_id: crate::atp::object::ObjectId::content(
crate::atp::object::ContentId::from_bytes(b"bad.bin"),
),
content_sha256: [0; 32],
}];
let pending = BTreeSet::from([0_u32]);
state.aimd_rate_bps = RQ_COLD_START_PACING_BPS;
state.observe_need_more(
&config,
&digests,
&pending,
10_000,
5_000,
Some(0.50),
RqDeliverySampleKind::InitialOrRepair,
Duration::from_secs(1),
Duration::from_millis(80),
total_bytes,
);
assert_eq!(state.last_round_loss_fraction, 0.50);
assert!(
state.aimd_rate_bps.abs_diff(6_600_000) <= 1,
"loss-targeted overrun should back off to receiver delivery plus headroom, not blind half-rate"
);
assert!(
state.aimd_rate_bps < RQ_COLD_START_PACING_BPS / 2,
"delivery-rate backoff must be able to settle below blind half-rate on sub-100mbit paths"
);
}
#[test]
fn rq_aimd_keeps_cold_start_floor_for_good_source_first_feedback() {
let config = RqConfig {
symbol_size: 1200,
round0_loss_target: 0.001,
..RqConfig::default()
};
let mut state = RqAdaptiveSendState::new(7, &config, 1);
let total_bytes = 500_u64 * 1024 * 1024;
let digests = [EntryDigest {
rel_path: "good.bin".to_string(),
size: total_bytes,
content_id: crate::atp::object::ObjectId::content(
crate::atp::object::ContentId::from_bytes(b"good.bin"),
),
content_sha256: [0; 32],
}];
let pending = BTreeSet::from([0_u32]);
state.observe_need_more(
&config,
&digests,
&pending,
10_000,
9_400,
Some(0.06),
RqDeliverySampleKind::InitialOrRepair,
Duration::from_millis(800),
Duration::from_millis(25),
total_bytes,
);
assert_eq!(state.last_round_loss_fraction, 0.06);
assert_eq!(
state.aimd_rate_bps, RQ_COLD_START_PACING_BPS,
"good-link source-first feedback should not halve below cold-start"
);
assert!(
state.loss_bar >= RQ_PENDING_PRESSURE_LOSS_FLOOR,
"repair sizing should still see pending pressure"
);
let tuning = state.round_tuning(&config);
assert_eq!(
tuning.pacing.path_rate_bps,
RQ_COLD_START_PACING_BPS.saturating_mul(8),
"feedback spray should keep the cold-start sender floor"
);
}
#[test]
fn rq_aimd_additively_increases_on_clean_receiver_round() {
let config = RqConfig {
symbol_size: 1200,
..RqConfig::default()
};
let mut state = RqAdaptiveSendState::new(7, &config, 1);
let total_bytes = 5_u64 * 1024 * 1024;
let digests = [EntryDigest {
rel_path: "clean.bin".to_string(),
size: total_bytes,
content_id: crate::atp::object::ObjectId::content(
crate::atp::object::ContentId::from_bytes(b"clean.bin"),
),
content_sha256: [0; 32],
}];
let pending = BTreeSet::from([0_u32]);
state.aimd_rate_bps = 4 * 1024 * 1024;
state.observe_need_more(
&config,
&digests,
&pending,
4_000,
4_000,
None,
RqDeliverySampleKind::InitialOrRepair,
Duration::from_millis(400),
Duration::from_millis(100),
total_bytes,
);
assert_eq!(state.last_round_loss_fraction, 0.0);
assert_eq!(
state.aimd_rate_bps,
4 * 1024 * 1024 + RQ_AIMD_ADDITIVE_INCREASE_BYTES_PER_S
);
}
#[test]
fn rq_round_tuning_honors_persistent_aimd_cap() {
let config = RqConfig::default();
let mut state = RqAdaptiveSendState::new(7, &config, 1);
state.est = rq_test_path_estimate(&config, RQ_MAX_PACING_BPS as f64);
state.controller.update_estimate(state.est);
state.aimd_rate_bps = 4 * 1024 * 1024;
state.aimd_feedback_seen = true;
let tuning = state.round_tuning(&config);
assert_eq!(
tuning.pacing.path_rate_bps,
state.aimd_rate_bps.saturating_mul(8),
"AIMD decrease must cap the next sender spray instead of acting as a one-shot sample"
);
}
#[test]
fn rq_shared_rate_decision_caps_next_pacer_burst() {
let config = RqConfig {
symbol_size: 1200,
round0_loss_target: 0.10,
..RqConfig::default()
};
let mut state = RqAdaptiveSendState::new(7, &config, 1);
let total_bytes = u64::from(config.symbol_size) * 2;
let digests = [EntryDigest {
rel_path: "receiver-credit.bin".to_string(),
size: total_bytes,
content_id: crate::atp::object::ObjectId::content(
crate::atp::object::ContentId::from_bytes(b"receiver-credit"),
),
content_sha256: [0; 32],
}];
let pending = BTreeSet::from([0_u32]);
state.observe_need_more(
&config,
&digests,
&pending,
10_000,
1_000,
Some(0.0),
RqDeliverySampleKind::InitialOrRepair,
Duration::from_secs(1),
Duration::from_millis(100),
total_bytes,
);
let decision = state
.shared_rate_decision()
.expect("NeedMore should feed shared datagram-rate controller");
let expected_credit = rq_receiver_flow_credit_bytes(&config, total_bytes);
assert!(
decision.sender_loss_fraction_ppm >= 900_000,
"shared controller must see sender-side queue overflow"
);
assert_eq!(
decision.receiver_credit_bytes,
Some(expected_credit),
"pending receiver bytes should be translated into the controller receiver credit"
);
assert_eq!(
decision.receiver_window_bytes,
Some(expected_credit.saturating_add(decision.bytes_in_flight)),
"receiver window should include still-outstanding sender payload plus remaining credit"
);
assert!(
decision.send_budget_bytes <= expected_credit,
"receiver credit must clip the immediate shared send budget"
);
let tuning = state.round_tuning(&config);
assert!(
tuning.pacing.rate_bytes_per_sec() <= decision.pacing_bytes_per_s,
"RQ round tuning must consume the shared controller's pacing cap"
);
let mut pacer = RqSprayPacer::new_round0(tuning.pacing, &config, false);
pacer.configure_with_shared_decision(tuning.pacing, Some(decision));
let now = Instant::now();
let mut immediate_budget = 0u32;
while pacer.controller.try_consume_send_budget(now) {
immediate_budget = immediate_budget.saturating_add(1);
assert!(
immediate_budget <= 2,
"receiver-limited shared decision reopened burst {immediate_budget}"
);
}
assert!(
immediate_budget <= 2,
"shared receiver credit should bound immediate RQ burst, got {immediate_budget}"
);
}
#[test]
fn rq_round_tuning_applies_loss_detector_cap_after_aimd_feedback() {
let config = RqConfig {
round0_loss_target: 0.02,
..RqConfig::default()
};
let mut state = RqAdaptiveSendState::new(7, &config, 1);
state.est = rq_test_path_estimate(&config, 32.0 * 1024.0 * 1024.0);
state.controller.update_estimate(state.est);
state.aimd_feedback_seen = true;
state.aimd_rate_bps = RQ_COLD_START_PACING_BPS;
state.loss_ema = 0.01;
state.loss_bar = RQ_PENDING_PRESSURE_LOSS_FLOOR;
state.pacing_loss_ema = config.round0_loss_target;
state.loss_pacing_cap_bps = Some(RQ_COLD_START_PACING_BPS / 4);
let tuning = state.round_tuning(&config);
assert_eq!(
tuning.pacing.rate_bytes_per_sec(),
RQ_BAD_LINK_ROUND0_PACING_BPS,
"loss-detector caps must remain active after AIMD feedback, bounded by the bad-link floor"
);
}
#[test]
fn rq_need_more_entry_pressure_does_not_create_congestion_cap() {
let config = RqConfig {
symbol_size: 1024,
..RqConfig::default()
};
let mut state = RqAdaptiveSendState::new(7, &config, 1);
let total_bytes = 5_u64 * 1024 * 1024;
let digests = [EntryDigest {
rel_path: "large.bin".to_string(),
size: total_bytes,
content_id: crate::atp::object::ObjectId::content(
crate::atp::object::ContentId::from_bytes(b"large.bin"),
),
content_sha256: [0; 32],
}];
let pending = BTreeSet::from([0_u32]);
let sent_symbols = total_bytes / u64::from(config.symbol_size);
state.observe_need_more(
&config,
&digests,
&pending,
sent_symbols,
sent_symbols.saturating_sub(1),
None,
RqDeliverySampleKind::InitialOrRepair,
Duration::from_secs(120),
Duration::from_millis(50),
total_bytes,
);
assert!(
state.loss_bar >= RQ_PENDING_PRESSURE_LOSS_FLOOR,
"large residual entry should still raise the FEC sizing floor"
);
assert!(
state.pacing_loss_ema <= RQ_MILD_LOSS_PACING_MAX_LOSS,
"entry-level residual pressure must not masquerade as pacing loss"
);
assert_eq!(
state.est.loss_p_bar, state.loss_bar,
"adaptive path estimate must preserve pending-aware FEC pressure"
);
assert!(
state.pacing_loss_bar < state.loss_bar,
"wire-loss pacing bar must remain below pending-aware FEC pressure"
);
assert_eq!(
state.loss_pacing_cap_bps, None,
"one residual pending entry must not manufacture a congestion cap"
);
let rate = state.pacing_rate_for(rq_test_block_plan(&config), &config);
assert!(
rate >= RQ_COLD_START_PACING_BPS / 2,
"mild residual repair should recover from the slow-sample floor, got {rate} B/s"
);
}
#[test]
fn rq_need_more_mild_wire_loss_keeps_pending_pressure_out_of_pacing() {
let config = RqConfig {
symbol_size: 1200,
..RqConfig::default()
};
let mut state = RqAdaptiveSendState::new(7, &config, 1);
let total_bytes = 50_u64 * 1024 * 1024;
let digests = [EntryDigest {
rel_path: "large.bin".to_string(),
size: total_bytes,
content_id: crate::atp::object::ObjectId::content(
crate::atp::object::ContentId::from_bytes(b"large.bin"),
),
content_sha256: [0; 32],
}];
let pending = BTreeSet::from([0_u32]);
for (sent_symbols, send_wall) in [
(43_700_u64, Duration::from_millis(3_300)),
(15_992, Duration::from_millis(12_300)),
(15_992, Duration::from_millis(13_300)),
(7_800, Duration::from_millis(12_000)),
(7_800, Duration::from_millis(13_800)),
(7_800, Duration::from_millis(15_000)),
] {
let lost_symbols = sent_symbols.div_ceil(50);
state.observe_need_more(
&config,
&digests,
&pending,
sent_symbols,
sent_symbols.saturating_sub(lost_symbols),
None,
RqDeliverySampleKind::InitialOrRepair,
send_wall,
Duration::from_millis(200),
total_bytes,
);
}
assert!(
state.loss_bar >= RQ_PENDING_PRESSURE_LOSS_FLOOR,
"pending pressure should remain available for FEC sizing"
);
assert!(
state.pacing_loss_ema <= RQ_MILD_LOSS_PACING_MAX_LOSS,
"2% receiver-observed wire loss should keep the mild pacing floor active"
);
assert!(
state.mild_loss_pacing_floor_applies(),
"entry-granular pending pressure must not disable the pacing floor"
);
assert_eq!(
state.est.loss_p_bar, state.loss_bar,
"PathEstimate loss bar must keep pending-aware FEC pressure for repair sizing"
);
assert!(
state.pacing_loss_bar < state.loss_bar,
"receiver-observed wire loss may stay mild while FEC pressure remains high"
);
assert!(
state.pacing_rate_for(rq_test_block_plan(&config), &config)
>= state.mild_loss_pacing_floor_bps(&config),
"stalled repair rounds must not drag pacing below the mild-loss floor"
);
}
#[test]
fn rq_source_retransmit_undercredit_does_not_poison_pacing_estimator() {
let config = RqConfig {
symbol_size: 1200,
..RqConfig::default()
};
let mut state = RqAdaptiveSendState::new(7, &config, 1);
let total_bytes = 50_u64 * 1024 * 1024;
let digests = [EntryDigest {
rel_path: "large.bin".to_string(),
size: total_bytes,
content_id: crate::atp::object::ObjectId::content(
crate::atp::object::ContentId::from_bytes(b"large.bin"),
),
content_sha256: [0; 32],
}];
let pending = BTreeSet::from([0_u32]);
state.observe_need_more(
&config,
&digests,
&pending,
15_992,
1_884,
None,
RqDeliverySampleKind::SourceRetransmit,
Duration::from_millis(12_300),
Duration::from_millis(200),
total_bytes,
);
assert_eq!(
state.last_round_loss_fraction, 0.0,
"under-credited source retransmit rounds must not become measured wire loss"
);
assert_eq!(
state.pacing_loss_ema, 0.0,
"source retransmit under-credit must stay out of pacing loss"
);
assert_eq!(
state.pacing_loss_bar, 0.0,
"source retransmit under-credit must not disable the pacing floor"
);
assert_eq!(
state.loss_pacing_cap_bps, None,
"source retransmit under-credit must not create a congestion cap"
);
assert!(
state.loss_bar >= RQ_PENDING_PRESSURE_LOSS_FLOOR,
"pending pressure should still size repair FEC"
);
assert_eq!(
state.round_tuning(&config).pacing.rate_bytes_per_sec(),
RQ_COLD_START_PACING_BPS,
"source retransmit under-credit must not collapse the next path rate"
);
}
#[test]
fn rq_mixed_source_and_fec_feedback_feeds_pacing_estimator() {
assert_eq!(
delivery_sample_kind_for_need_more_response(16, false),
RqDeliverySampleKind::SourceRetransmit,
"pure source retransmit under-credit should stay out of pacing loss"
);
assert_eq!(
delivery_sample_kind_for_need_more_response(16, true),
RqDeliverySampleKind::InitialOrRepair,
"source retransmit plus FEC repair must feed receiver-observed wire loss into AIMD"
);
assert_eq!(
delivery_sample_kind_for_need_more_response(0, false),
RqDeliverySampleKind::InitialOrRepair,
"repair-only feedback rounds are pacing samples"
);
}
#[test]
fn rq_need_more_broken_wire_loss_disables_mild_floor() {
let config = RqConfig {
symbol_size: 1200,
..RqConfig::default()
};
let mut state = RqAdaptiveSendState::new(7, &config, 1);
let total_bytes = 50_u64 * 1024 * 1024;
let digests = [EntryDigest {
rel_path: "large.bin".to_string(),
size: total_bytes,
content_id: crate::atp::object::ObjectId::content(
crate::atp::object::ContentId::from_bytes(b"large.bin"),
),
content_sha256: [0; 32],
}];
let pending = BTreeSet::from([0_u32]);
let sent_symbols = 43_700_u64;
state.observe_need_more(
&config,
&digests,
&pending,
sent_symbols,
sent_symbols.saturating_sub(sent_symbols.div_ceil(10)),
None,
RqDeliverySampleKind::InitialOrRepair,
Duration::from_millis(3_300),
Duration::from_millis(200),
total_bytes,
);
assert!(
state.pacing_loss_ema > RQ_MILD_LOSS_PACING_MAX_LOSS,
"10% receiver-observed wire loss should exceed the mild-loss threshold"
);
assert!(
!state.mild_loss_pacing_floor_applies(),
"broken-link wire loss must still allow conservative pacing"
);
assert!(
state.loss_bar >= RQ_PENDING_PRESSURE_LOSS_FLOOR,
"broken-link pending pressure should still size repair FEC"
);
}
#[test]
fn rq_pending_send_batch_groups_by_round_robin_socket() {
let mut batch = RqPendingSendBatch::new(4);
let global_flush_symbols = batch.global_flush_symbols();
assert_eq!(global_flush_symbols, RQ_SEND_BATCH_PER_SOCKET);
for i in 0..global_flush_symbols {
batch.push(i % 4, vec![u8::try_from(i).unwrap_or(u8::MAX)]);
}
assert_eq!(batch.queued_count(), global_flush_symbols);
assert!(batch.should_flush());
assert_eq!(batch.socket_batch_len(0), RQ_SEND_BATCH_PER_SOCKET / 4);
assert_eq!(batch.socket_batch_len(1), RQ_SEND_BATCH_PER_SOCKET / 4);
assert_eq!(batch.socket_batch_len(2), RQ_SEND_BATCH_PER_SOCKET / 4);
assert_eq!(batch.socket_batch_len(3), RQ_SEND_BATCH_PER_SOCKET / 4);
}
#[test]
fn rq_pending_send_batch_caps_aggregate_burst_across_fanout() {
let mut batch = RqPendingSendBatch::new(8);
let almost_full = batch.global_flush_symbols().saturating_sub(1);
for i in 0..almost_full {
batch.push(i % batch.fanout(), vec![u8::try_from(i).unwrap_or(u8::MAX)]);
}
assert!(!batch.should_flush());
assert!(
batch
.by_socket
.iter()
.all(|payloads| payloads.len() < RQ_SEND_BATCH_PER_SOCKET)
);
batch.push(almost_full % batch.fanout(), vec![0]);
assert!(batch.should_flush());
assert_eq!(batch.queued_count(), RQ_SEND_BATCH_PER_SOCKET);
assert!(
batch
.by_socket
.iter()
.all(|payloads| payloads.len() <= RQ_SEND_BATCH_PER_SOCKET.div_ceil(8))
);
}
#[test]
fn rq_pending_send_batch_flushes_on_single_socket_bound() {
let mut batch = RqPendingSendBatch::new(4);
for i in 0..RQ_SEND_BATCH_PER_SOCKET {
batch.push(0, vec![u8::try_from(i).unwrap_or(u8::MAX)]);
}
assert_eq!(batch.queued_count(), RQ_SEND_BATCH_PER_SOCKET);
assert!(batch.should_flush());
assert_eq!(batch.socket_batch_len(0), RQ_SEND_BATCH_PER_SOCKET);
assert_eq!(batch.socket_batch_len(1), 0);
}
#[test]
fn rq_default_authenticated_datagrams_plan_as_one_gso_super_packet() {
let dst_addr: SocketAddr = "127.0.0.1:9000".parse().expect("socket address");
let ctx = SecurityContext::for_testing(77);
let payloads = (0..RQ_SEND_BATCH_PER_SOCKET)
.map(|esi| {
let sym = Symbol::new(
SymbolId::new(
ObjectId::new(1, 2),
0,
u32::try_from(esi).expect("test ESI fits u32"),
),
vec![u8::try_from(esi).unwrap_or(u8::MAX); usize::from(DEFAULT_SYMBOL_SIZE)],
SymbolKind::Source,
);
let auth = ctx.sign_symbol(&sym);
encode_symbol_datagram(0xABCD, 0, &sym, Some(auth.tag()))
})
.collect::<Vec<_>>();
assert!(
payloads.iter().all(|payload| {
payload.len() == AUTH_DGRAM_HEADER + usize::from(DEFAULT_SYMBOL_SIZE)
}),
"default authenticated RQ datagrams must keep fixed GSO segment size",
);
let packets = payloads
.iter()
.map(|payload| crate::net::UdpOutboundDatagram { dst_addr, payload })
.collect::<Vec<_>>();
let plan = crate::net::UdpSendBatchPlan::for_packets(
&packets,
crate::net::UdpSendAccelerationCapabilities {
sendmmsg: crate::net::UdpCapability::Supported,
gso: crate::net::UdpCapability::Supported,
max_sendmmsg_batch: crate::net::UDP_MAX_SENDMMSG_BATCH,
max_gso_segments: crate::net::UDP_MAX_GSO_SEGMENTS,
},
crate::net::UdpSendBatchStrategy::default(),
);
assert_eq!(plan.path, crate::net::UdpSendBatchPath::Gso);
assert_eq!(plan.estimated_syscalls, 1);
assert_eq!(plan.gso_segments_per_packet, Some(RQ_SEND_BATCH_PER_SOCKET));
assert_eq!(
plan.gso_segment_bytes,
Some(AUTH_DGRAM_HEADER + usize::from(DEFAULT_SYMBOL_SIZE))
);
}
#[test]
fn rq_batched_send_yields_when_progress_crosses_boundary() {
assert!(!send_progress_crossed_yield_boundary(0, 63));
assert!(send_progress_crossed_yield_boundary(63, 64));
assert!(send_progress_crossed_yield_boundary(60, 96));
assert!(!send_progress_crossed_yield_boundary(64, 96));
}
#[test]
fn rq_sender_window_probe_estimates_bdp_limited_window() {
let pacing = RqSprayPacing::from_rate(
16 * 1024 * 1024,
1000,
RQ_COLD_START_BURST_SYMBOLS,
None,
false,
);
let probe = RqSenderWindowProbe::new(
pacing,
13_200,
1000,
Duration::from_secs(1),
Duration::from_millis(200),
);
assert_eq!(probe.payload_bytes, 13_200_000);
assert_eq!(probe.observed_payload_bytes_per_sec, 13_200_000);
assert_eq!(probe.observed_payload_window_bytes, 2_640_000);
assert_eq!(probe.configured_rate_bytes_per_sec, 16 * 1024 * 1024);
assert_eq!(probe.configured_bdp_bytes, 0);
assert_eq!(
probe.configured_control_window_bytes,
rate_window_bytes(16 * 1024 * 1024, Duration::from_millis(200))
);
assert_eq!(
probe.peak_window_bytes(),
probe
.configured_bdp_bytes
.max(probe.configured_control_window_bytes)
.max(probe.observed_payload_window_bytes)
.max(probe.observed_wire_window_bytes)
);
}
#[test]
fn rq_loss_recommendations_apply_advisory_caps_and_fec_floor() {
let config = RqConfig::default();
let mut state = RqAdaptiveSendState::new(7, &config, 1);
state.bw_ema_bps = 10_000_000.0;
state.apply_loss_recommendations(
&[
LossRecommendation::ReduceCongestionWindow { factor: 0.5 },
LossRecommendation::EnableFec { rate: 0.10 },
LossRecommendation::SwitchCongestionControl {
algorithm: "bbr".to_string(),
},
],
false,
);
assert_eq!(state.loss_pacing_cap_bps, Some(5_000_000));
assert!(state.loss_fec_floor >= 0.10);
assert!(state.regime_shift);
}
#[test]
fn rq_loss_recommendations_keep_wire_rate_under_expected_regime_loss() {
let config = RqConfig::default();
let mut state = RqAdaptiveSendState::new(7, &config, 1);
state.bw_ema_bps = 10_000_000.0;
state.apply_loss_recommendations(
&[
LossRecommendation::ReduceCongestionWindow { factor: 0.5 },
LossRecommendation::EnablePacing { rate: 1_000_000 },
LossRecommendation::EnableFec { rate: 0.10 },
],
true,
);
assert_eq!(
state.loss_pacing_cap_bps, None,
"loss at or below the regime's expectation is the ambient erasure \
condition FEC pays for, not congestion — the pacing cap must not \
drop (500M/broken repair-round collapse, MATRIX-207)"
);
assert!(
state.loss_fec_floor >= 0.10,
"FEC sizing must stay available under expected loss"
);
assert!(!state.regime_shift);
}
#[test]
fn rq_feedback_bandwidth_uses_send_wall_not_feedback_wait() {
let config = RqConfig::default();
let mut state = RqAdaptiveSendState::new(23, &config, 1);
let total_bytes = 5 * 1024 * 1024_u64;
let sent_symbols = total_bytes.div_ceil(u64::from(config.symbol_size.max(1)));
let digests = vec![EntryDigest {
rel_path: "payload.bin".to_string(),
size: total_bytes,
content_id: crate::atp::object::ObjectId::content(
crate::atp::object::ContentId::from_bytes(b"rq-feedback-bandwidth"),
),
content_sha256: [0x42; 32],
}];
let pending = BTreeSet::from([0]);
state.observe_need_more(
&config,
&digests,
&pending,
sent_symbols,
sent_symbols,
None,
RqDeliverySampleKind::InitialOrRepair,
Duration::from_millis(300),
Duration::from_secs(120),
total_bytes,
);
assert!(
state.bw_ema_bps > 8.0 * 1024.0 * 1024.0,
"feedback timeout must not collapse a fast spray sample to {} B/s",
state.bw_ema_bps
);
}
/// In-process encode→feed→decode roundtrip at a chosen `(bytes, max_block)`,
/// mirroring exactly how `spray_round` encodes and `feed_symbol` decodes —
/// but with NO network — so a coding/params mismatch is isolated from the
/// transport. Feeds source + a generous repair tail and asserts the block
/// decodes back to the original bytes.
fn coding_roundtrip(len: usize, max_block: usize, symbol_size: u16) -> bool {
let bytes: Vec<u8> = (0..len)
.map(|i| (i.wrapping_mul(2654435761) >> 13) as u8)
.collect();
let object_id = entry_object_id("test-transfer", 0);
// Encode: source + repair (generous), like spray_round.
let block_k = max_block.div_ceil(usize::from(symbol_size.max(1))).max(1);
let repair = block_k; // 100% repair — far more than needed
let pool = SymbolPool::new(PoolConfig::default());
let mut enc = EncodingPipeline::new(
crate::config::EncodingConfig {
repair_overhead: 1.5,
max_block_size: max_block,
symbol_size,
encoding_parallelism: 1,
decoding_parallelism: 1,
},
pool,
);
let symbols: Vec<Symbol> = enc
.encode_with_repair(object_id, &bytes, repair)
.map(|e| e.unwrap().into_symbol())
.collect();
// Decode: feed all symbols, like feed_symbol.
let dconfig = DecodingConfig {
symbol_size,
max_block_size: max_block,
repair_overhead: 1.5,
min_overhead: 0,
max_buffered_symbols: 0,
block_timeout: std::time::Duration::from_secs(0),
verify_auth: false,
};
let mut dec = DecodingPipeline::new(dconfig);
let params = object_params_for(object_id, len as u64, symbol_size, max_block as u64);
dec.set_object_params(params).expect("set_object_params");
for s in symbols {
let _ = dec.feed(AuthenticatedSymbol::new_unauthenticated(s));
}
if !dec.is_complete() {
return false;
}
let mut out = dec.into_data().expect("into_data");
out.truncate(len);
out == bytes
}
#[test]
fn coding_roundtrip_small_k_single_block() {
// K = 64 (matches the loopback e2e regime).
assert!(coding_roundtrip(60_000, 64 * 1024, 1024));
}
#[test]
fn coding_roundtrip_k512_single_block() {
// K = 512 single 8 MiB block — the default-config regime that the
// cross-machine transfer exercised. Regression guard for the
// never-converges bug.
assert!(coding_roundtrip(512 * 1024, 8 * 1024 * 1024, 1024));
}
#[test]
fn coding_roundtrip_multi_block_small_k() {
// Three 64 KiB blocks at K=64 exercises SBN routing, per-block decode,
// and final cross-block assembly without making the normal unit lane
// pay the K=1024 matrix cost of the historical fleet repro.
assert!(coding_roundtrip(3 * 64 * 1024, 64 * 1024, 1024));
}
#[test]
fn default_k_multiblock_metadata_is_accepted_by_decoder() {
// Regression guard for br-asupersync-c8m8ha: the default-ish multi-block
// shape used to fail at set_object_params before any network I/O. Keep
// this as metadata-only coverage so the guard stays cheap and stable.
let len = 3 * 1024 * 1024;
let max_block = 1024 * 1024;
let symbol_size = 1024;
let object_id = entry_object_id("test-transfer", 0);
let dconfig = DecodingConfig {
symbol_size,
max_block_size: max_block,
repair_overhead: 1.5,
min_overhead: 0,
max_buffered_symbols: 0,
block_timeout: std::time::Duration::from_secs(0),
verify_auth: false,
};
let mut dec = DecodingPipeline::new(dconfig);
let params = object_params_for(object_id, len as u64, symbol_size, max_block as u64);
assert_eq!(params.source_blocks, 3);
assert_eq!(params.symbols_per_block, 1024);
dec.set_object_params(params)
.expect("default-ish multi-block params must match decoder plan");
}
#[test]
fn safe_base_for_root_name_contains_hostile_inputs() {
// Regression guard: a malicious sender controls `root_name` off the
// wire. It must never escape `dest_dir`, even when absolute or
// separator-bearing (Path::join replaces the base for absolute args).
let dest = Path::new("/dst");
assert_eq!(
safe_base_for_root_name(dest, "payload").unwrap(),
dest.join("payload")
);
// Absolute root_name would otherwise replace the base via Path::join;
// collapse to the final component instead.
assert_eq!(
safe_base_for_root_name(dest, "/etc/cron.d/evil").unwrap(),
dest.join("evil")
);
assert_eq!(
safe_base_for_root_name(dest, "../../etc/passwd").unwrap(),
dest.join("passwd")
);
assert!(safe_base_for_root_name(dest, "").is_err());
assert!(safe_base_for_root_name(dest, "/").is_err());
assert!(safe_base_for_root_name(dest, "..").is_err());
}
fn manifest_with(entries: Vec<ManifestEntry>, total_bytes: u64) -> TransferManifest {
TransferManifest {
transfer_id: "rqtransfer1".to_string(),
root_name: "payload".to_string(),
is_directory: true,
total_bytes,
merkle_root_hex: "0".repeat(64),
entries,
}
}
fn manifest_entry(index: u32, size: u64) -> ManifestEntry {
ManifestEntry {
index,
rel_path: format!("f{index}"),
size,
sha256_hex: "0".repeat(64),
members: Vec::new(),
fragment: None,
}
}
#[test]
fn manifest_entry_members_serde_backward_compat() {
// E-15 S1: a pre-packing manifest entry (no `members`) must deserialize to empty
// members AND re-serialize WITHOUT a `members` field, so the no-packing wire stays
// byte-identical to before E-15.
let old_json = r#"{"index":0,"rel_path":"f0","size":10,"sha256_hex":"00"}"#;
let parsed: ManifestEntry =
serde_json::from_str(old_json).expect("deserialize pre-packing entry");
assert!(parsed.members.is_empty(), "missing members => empty");
let reser = serde_json::to_string(&parsed).expect("serialize");
assert!(
!reser.contains("members"),
"empty members must be skipped (byte-identical no-packing wire): {reser}"
);
// A packed entry round-trips with its member offset table intact.
let packed = ManifestEntry {
index: 1,
rel_path: ".atp-pack-0".to_string(),
size: 20,
sha256_hex: "ab".repeat(32),
members: vec![
PackedMember {
rel_path: "dir/a".to_string(),
offset: 0,
len: 10,
sha256_hex: "aa".repeat(32),
},
PackedMember {
rel_path: "dir/b".to_string(),
offset: 10,
len: 10,
sha256_hex: "bb".repeat(32),
},
],
fragment: None,
};
let json = serde_json::to_string(&packed).expect("serialize packed");
assert!(json.contains("members"), "packed entry serializes members");
let back: ManifestEntry = serde_json::from_str(&json).expect("round-trip packed");
assert_eq!(back, packed, "packed entry round-trips byte-identical");
}
#[test]
fn validate_manifest_accepts_sane_bounds() {
let manifest = manifest_with(vec![manifest_entry(0, 100), manifest_entry(1, 200)], 300);
assert!(validate_manifest(&manifest, &RqConfig::default()).is_ok());
}
#[test]
fn parse_manifest_frame_validates_before_receiver_state() {
let mut entries = vec![manifest_entry(0, 10), manifest_entry(1, 20)];
entries[1].rel_path = entries[0].rel_path.clone();
let manifest = manifest_with(entries, 30);
let frame = json_frame(FrameType::ObjectManifest, &manifest).expect("manifest frame");
assert!(matches!(
parse_and_validate_manifest_frame(&frame, &RqConfig::default()),
Err(RqError::Frame(msg)) if msg.contains("duplicate manifest rel_path")
));
}
#[test]
fn validate_manifest_rejects_lying_entry_size() {
let manifest = manifest_with(vec![manifest_entry(0, u64::MAX)], 10);
assert!(matches!(
validate_manifest(&manifest, &RqConfig::default()),
Err(RqError::TooLarge { .. })
));
}
#[test]
fn validate_manifest_rejects_declared_sum_over_limit() {
let config = RqConfig {
max_transfer_bytes: 1000,
..RqConfig::default()
};
let manifest = manifest_with(vec![manifest_entry(0, 600), manifest_entry(1, 600)], 1200);
assert!(matches!(
validate_manifest(&manifest, &config),
Err(RqError::TooLarge { .. })
));
}
#[test]
fn validate_manifest_rejects_declared_sum_overflow() {
let config = RqConfig {
max_transfer_bytes: u64::MAX,
..RqConfig::default()
};
let manifest = manifest_with(
vec![manifest_entry(0, u64::MAX), manifest_entry(1, 1)],
u64::MAX,
);
assert!(matches!(
validate_manifest(&manifest, &config),
Err(RqError::Frame(msg)) if msg.contains("declared size sum overflows")
));
}
#[test]
fn validate_manifest_rejects_single_file_with_multiple_entries() {
let mut manifest = manifest_with(vec![manifest_entry(0, 10), manifest_entry(1, 20)], 30);
manifest.is_directory = false;
assert!(matches!(
validate_manifest(&manifest, &RqConfig::default()),
Err(RqError::Frame(msg)) if msg.contains("single-file transfer")
));
}
#[test]
fn validate_manifest_rejects_duplicate_relative_paths() {
let mut entries = vec![manifest_entry(0, 10), manifest_entry(1, 20)];
entries[1].rel_path = entries[0].rel_path.clone();
let manifest = manifest_with(entries, 30);
assert!(matches!(
validate_manifest(&manifest, &RqConfig::default()),
Err(RqError::Frame(msg)) if msg.contains("duplicate manifest rel_path")
));
}
#[test]
fn validate_manifest_rejects_nonsequential_indexes() {
let manifest = manifest_with(vec![manifest_entry(0, 10), manifest_entry(7, 20)], 30);
assert!(matches!(
validate_manifest(&manifest, &RqConfig::default()),
Err(RqError::Frame(msg)) if msg.contains("does not match position")
));
}
#[test]
fn validate_manifest_rejects_unsafe_relative_paths() {
for rel_path in [
"",
"/abs",
"\\abs",
"../escape",
"a/../escape",
"a//b",
"a\\b",
"c:drive",
] {
let mut entry = manifest_entry(0, 10);
entry.rel_path = rel_path.to_string();
let manifest = manifest_with(vec![entry], 10);
assert!(
matches!(
validate_manifest(&manifest, &RqConfig::default()),
Err(RqError::Source(msg)) if msg.contains("unsafe manifest rel_path")
),
"rel_path {rel_path:?} should fail closed"
);
}
}
#[test]
fn validate_manifest_rejects_unsafe_transfer_id() {
let long = "x".repeat(65);
for transfer_id in ["", "../escape", "with/slash", "with-hyphen", long.as_str()] {
let mut manifest = manifest_with(vec![manifest_entry(0, 10)], 10);
manifest.transfer_id = transfer_id.to_string();
assert!(
matches!(
validate_manifest(&manifest, &RqConfig::default()),
Err(RqError::Frame(msg)) if msg.contains("unsafe manifest transfer_id")
),
"transfer_id {transfer_id:?} should fail closed"
);
}
}
#[test]
fn validate_manifest_rejects_malformed_hash_fields() {
let mut bad_root = manifest_with(vec![manifest_entry(0, 10)], 10);
bad_root.merkle_root_hex = "0".repeat(63);
assert!(matches!(
validate_manifest(&bad_root, &RqConfig::default()),
Err(RqError::Frame(msg)) if msg.contains("manifest merkle_root_hex")
));
let mut bad_entry = manifest_entry(0, 10);
bad_entry.sha256_hex = "zz".repeat(32);
assert!(matches!(
validate_manifest(&manifest_with(vec![bad_entry], 10), &RqConfig::default()),
Err(RqError::Frame(msg)) if msg.contains("manifest entry sha256_hex")
));
let mut bad_fragment = ManifestEntry {
index: 0,
rel_path: ".atp-fragment-0-0".to_string(),
size: 10,
sha256_hex: "00".repeat(32),
members: Vec::new(),
fragment: Some(LargeObjectFragment {
rel_path: "huge.bin".to_string(),
shard_index: 0,
shard_count: 1,
logical_offset: 0,
len: 10,
logical_size: 10,
sha256_hex: "f".repeat(63),
}),
};
let mut fragment_manifest = manifest_with(vec![bad_fragment.clone()], 10);
fragment_manifest.is_directory = false;
assert!(matches!(
validate_manifest(&fragment_manifest, &RqConfig::default()),
Err(RqError::Frame(msg)) if msg.contains("manifest fragment sha256_hex")
));
bad_fragment.fragment = None;
bad_fragment.rel_path = ".atp-pack-0".to_string();
bad_fragment.members = vec![PackedMember {
rel_path: "packed/member".to_string(),
offset: 0,
len: 10,
sha256_hex: "not-hex".to_string(),
}];
assert!(matches!(
validate_manifest(&manifest_with(vec![bad_fragment], 10), &RqConfig::default()),
Err(RqError::Frame(msg)) if msg.contains("manifest packed member sha256_hex")
));
}
#[cfg(unix)]
#[test]
fn rq_commit_rejects_existing_destination_symlink_prefix() {
let dest = tempfile::tempdir().expect("dest dir");
let outside = tempfile::tempdir().expect("outside dir");
let base = dest.path().join("payload");
std::fs::create_dir_all(&base).expect("create destination base");
std::os::unix::fs::symlink(outside.path(), base.join("link"))
.expect("create destination symlink");
let bytes = b"must stay inside the RQ destination".to_vec();
let staging_dir = dest.path().join(".atp-rq-test-staging");
std::fs::create_dir_all(&staging_dir).expect("create staging dir");
let staging_path = staging_dir.join("0");
std::fs::write(&staging_path, &bytes).expect("write staged payload");
let mut hash_buf = vec![0u8; RQ_STREAM_HASH_BUFFER_SIZE];
let (size, content_id, content_sha256) =
futures_lite::future::block_on(hash_file_streaming(&staging_path, &mut hash_buf))
.expect("hash staged payload");
let rel_path = "link/payload.txt".to_string();
let sha256_hex = hex_encode(&content_sha256);
let merkle_root_hex = flat_merkle_root_from_digests(&[EntryDigest {
rel_path: rel_path.clone(),
size,
content_id,
content_sha256,
}]);
let manifest = TransferManifest {
transfer_id: "rqtransfer1".to_string(),
root_name: "payload".to_string(),
is_directory: true,
total_bytes: size,
merkle_root_hex,
entries: vec![ManifestEntry {
index: 0,
rel_path,
size,
sha256_hex,
members: Vec::new(),
fragment: None,
}],
};
let mut decoders = vec![EntryDecoder {
index: 0,
object_id: entry_object_id(&manifest.transfer_id, 0),
size,
pipeline: None,
complete: true,
staging_path,
staging_write_offset: 0,
staging_file_len: size,
staging_shared: false,
staging_created: true,
staging_file: None,
staging_cursor: None,
staging_unflushed_bytes: 0,
cache_staging_file: false,
bytes_written: size,
max_block_size: DEFAULT_MAX_BLOCK_SIZE,
source_streaming: false,
source_blocks: Vec::new(),
pending_decodes: Vec::new(),
inc: None,
inc_digest: None,
source_write_buffer: Vec::new(),
source_write_buffer_offset: None,
}];
let err = futures_lite::future::block_on(verify_and_commit(
&manifest,
&mut decoders,
dest.path(),
0,
0,
&std::collections::BTreeMap::new(),
&CompletionDigestIndex::default(),
))
.expect_err("commit must reject pre-existing symlink ancestors");
assert!(
matches!(err, RqError::Source(ref message) if message.contains("existing symlink")),
"expected existing-symlink source error, got {err:?}"
);
assert!(
!outside.path().join("payload.txt").exists(),
"RQ commit must not follow a destination symlink outside dest_dir"
);
}
#[test]
fn datagram_roundtrips() {
let sym = Symbol::new(
SymbolId::new(ObjectId::new(1, 2), 3, 7),
vec![9u8; 1024],
SymbolKind::Repair,
);
let dg = encode_symbol_datagram(0xABCD, 42, &sym, None);
let parsed = parse_symbol_header(&dg, 0xABCD, false).expect("parse");
assert_eq!(parsed.entry, 42);
assert_eq!(parsed.sbn, 3);
assert_eq!(parsed.esi, 7);
assert!(matches!(parsed.kind, SymbolKind::Repair));
assert_eq!(parsed.auth_tag, None);
assert_eq!(parsed.payload_len, 1024);
assert_eq!(
&dg[parsed.header_len..parsed.header_len + 1024],
&[9u8; 1024]
);
}
#[test]
fn signed_datagram_roundtrips() {
let ctx = SecurityContext::for_testing(99);
let sym = Symbol::new(
SymbolId::new(ObjectId::new(1, 2), 3, 7),
vec![9u8; 1024],
SymbolKind::Repair,
);
let auth = ctx.sign_symbol(&sym);
let dg = encode_symbol_datagram(0xABCD, 42, &sym, Some(auth.tag()));
let parsed = parse_symbol_header(&dg, 0xABCD, true).expect("parse signed");
assert_eq!(parsed.entry, 42);
assert_eq!(parsed.sbn, 3);
assert_eq!(parsed.auth_tag, Some(*auth.tag()));
assert_eq!(parsed.header_len, AUTH_DGRAM_HEADER);
let mut received = AuthenticatedSymbol::from_parts(sym, parsed.auth_tag.expect("tag"));
ctx.verify_authenticated_symbol(&mut received)
.expect("tag verifies");
assert!(received.is_verified());
}
fn source_streaming_test_decoder(
object_id: ObjectId,
staging_path: PathBuf,
size: u64,
symbol_size: u16,
) -> EntryDecoder {
EntryDecoder {
index: 0,
object_id,
size,
pipeline: None,
complete: false,
staging_path,
staging_write_offset: 0,
staging_file_len: size,
staging_shared: false,
staging_created: false,
staging_file: None,
staging_cursor: None,
staging_unflushed_bytes: 0,
cache_staging_file: false,
bytes_written: 0,
max_block_size: usize::try_from(size).expect("test size fits usize"),
source_streaming: true,
source_blocks: source_block_progress_for(
size,
usize::try_from(size).expect("test size fits usize"),
symbol_size,
)
.expect("test source blocks"),
pending_decodes: Vec::new(),
inc: None,
inc_digest: None,
source_write_buffer: Vec::new(),
source_write_buffer_offset: None,
}
}
fn signed_source_payload(
ctx: &SecurityContext,
object_id: ObjectId,
esi: u32,
data: Vec<u8>,
tag: Option<AuthenticationTag>,
) -> (ParsedDatagram, Vec<u8>) {
let sym = Symbol::new(SymbolId::new(object_id, 0, esi), data, SymbolKind::Source);
let signed = ctx.sign_symbol(&sym);
let auth_tag = tag.as_ref().unwrap_or_else(|| signed.tag());
let dg = encode_symbol_datagram(0xA77E, 0, &sym, Some(auth_tag));
let parsed = parse_symbol_header(&dg, 0xA77E, true).expect("parse signed source datagram");
let payload = dg[parsed.header_len..parsed.header_len + parsed.payload_len].to_vec();
(parsed, payload)
}
fn signed_source_datagram(
ctx: &SecurityContext,
object_id: ObjectId,
esi: u32,
data: Vec<u8>,
tag: Option<AuthenticationTag>,
) -> Vec<u8> {
let sym = Symbol::new(SymbolId::new(object_id, 0, esi), data, SymbolKind::Source);
let signed = ctx.sign_symbol(&sym);
let auth_tag = tag.as_ref().unwrap_or_else(|| signed.tag());
encode_symbol_datagram(0xA77E, 0, &sym, Some(auth_tag))
}
fn udp_recv_batch(datagrams: Vec<Vec<u8>>) -> crate::net::UdpRecvBatch {
let src_addr = "127.0.0.1:9000".parse().expect("socket addr");
crate::net::UdpRecvBatch {
packets: datagrams
.into_iter()
.map(|payload| crate::net::UdpInboundDatagram {
src_addr,
payload,
possibly_truncated: false,
})
.collect(),
report: crate::net::UdpBatchIoReport::default(),
}
}
#[test]
fn plain_source_recv_batch_persists_without_pipeline_feed() {
let object_id = entry_object_id("plain-source-batch", 0);
let symbol_size = 4u16;
let dir = tempfile::tempdir().expect("tempdir");
let staging_path = dir.path().join("entry0");
let decoder =
source_streaming_test_decoder(object_id, staging_path.clone(), 8, symbol_size);
let first = Symbol::new(
SymbolId::new(object_id, 0, 0),
vec![1, 2, 3, 4],
SymbolKind::Source,
);
let second = Symbol::new(
SymbolId::new(object_id, 0, 1),
vec![5, 6, 7, 8],
SymbolKind::Source,
);
let batch = udp_recv_batch(vec![
encode_symbol_datagram(0xB47C, 0, &first, None),
encode_symbol_datagram(0xB47C, 0, &second, None),
]);
let cx = Cx::for_testing();
let mut decoders = vec![decoder];
let stats = futures_lite::future::block_on(feed_datagram_batch_to_decoders(
&cx,
&batch,
0xB47C,
false,
None,
&mut decoders,
symbol_size,
false,
))
.expect("feed plain source batch");
assert_eq!(stats.observed, 2);
assert_eq!(stats.accepted, 2);
assert_eq!(stats.payload_bytes, 8);
assert_eq!(stats.pipeline_feed_micros, 0);
assert_eq!(stats.decode_stats.attempts, 0);
assert!(decoders[0].complete);
assert_eq!(decoders[0].bytes_written, 8);
assert_eq!(
std::fs::read(staging_path).expect("read batch-staged source stream"),
vec![1, 2, 3, 4, 5, 6, 7, 8]
);
}
#[test]
fn signed_source_streaming_persists_after_hmac_verification() {
let ctx = SecurityContext::for_testing(31337);
let object_id = entry_object_id("signed-source-stream", 0);
let symbol_size = 4u16;
let dir = tempfile::tempdir().expect("tempdir");
let staging_path = dir.path().join("entry0");
let mut decoder = source_streaming_test_decoder(object_id, staging_path.clone(), 8, 4);
let (first, first_payload) =
signed_source_payload(&ctx, object_id, 0, vec![1, 2, 3, 4], None);
let accepted = futures_lite::future::block_on(feed_symbol(
&mut decoder,
&first,
&first_payload,
symbol_size,
Some(&ctx),
))
.expect("feed first source symbol");
assert!(accepted);
assert!(!decoder.complete);
assert!(decoder.staging_created);
assert_eq!(
std::fs::read(&staging_path).expect("read staged first source symbol"),
vec![1, 2, 3, 4, 0, 0, 0, 0]
);
let (second, second_payload) =
signed_source_payload(&ctx, object_id, 1, vec![5, 6, 7, 8], None);
let accepted = futures_lite::future::block_on(feed_symbol(
&mut decoder,
&second,
&second_payload,
symbol_size,
Some(&ctx),
))
.expect("feed second source symbol");
assert!(accepted);
assert!(decoder.complete);
assert_eq!(decoder.bytes_written, 8);
assert_eq!(
std::fs::read(staging_path).expect("read staged source stream"),
vec![1, 2, 3, 4, 5, 6, 7, 8]
);
}
#[test]
fn source_streaming_large_entry_cache_coalesces_and_closes_staging_file() {
let object_id = entry_object_id("source-stream-cache", 0);
let symbol_size = 4u16;
let dir = tempfile::tempdir().expect("tempdir");
let staging_path = dir.path().join("entry0");
let mut decoder =
source_streaming_test_decoder(object_id, staging_path.clone(), 8, symbol_size);
decoder.cache_staging_file = true;
let first = ParsedDatagram {
entry: 0,
sbn: 0,
esi: 0,
kind: SymbolKind::Source,
auth_tag: None,
payload_len: 4,
header_len: 0,
};
assert!(
futures_lite::future::block_on(persist_source_symbol(
&mut decoder,
&first,
&[1, 2, 3, 4],
symbol_size,
))
.expect("persist first cached source symbol")
);
assert!(
decoder.staging_file.is_none(),
"large-entry source streaming should buffer clean contiguous symbols before opening the staging file"
);
assert_eq!(decoder.staging_cursor, None);
assert_eq!(decoder.staging_unflushed_bytes, 0);
assert_eq!(decoder.source_write_buffer_offset, Some(0));
assert_eq!(decoder.source_write_buffer, vec![1, 2, 3, 4]);
let second = ParsedDatagram { esi: 1, ..first };
assert!(
futures_lite::future::block_on(persist_source_symbol(
&mut decoder,
&second,
&[5, 6, 7, 8],
symbol_size,
))
.expect("persist second cached source symbol")
);
assert!(decoder.complete);
assert!(
decoder.staging_file.is_none(),
"completed entries must release cached staging descriptors"
);
assert_eq!(decoder.staging_cursor, None);
assert_eq!(decoder.staging_unflushed_bytes, 0);
assert!(decoder.source_write_buffer.is_empty());
assert_eq!(decoder.source_write_buffer_offset, None);
assert_eq!(
std::fs::read(staging_path).expect("read cached source stream"),
vec![1, 2, 3, 4, 5, 6, 7, 8]
);
}
#[test]
fn plain_source_datagram_batch_persists_contiguous_run_once() {
let tag = 0xA77E_2024;
let object_id = entry_object_id("plain-source-batch-run", 0);
let symbol_size = 4u16;
let data: Vec<u8> = (1..=16).collect();
let dir = tempfile::tempdir().expect("tempdir");
let staging_path = dir.path().join("entry0");
let mut decoder =
source_streaming_test_decoder(object_id, staging_path.clone(), 16, symbol_size);
decoder.cache_staging_file = true;
let src_addr: SocketAddr = "127.0.0.1:31337".parse().expect("socket addr");
let packets = data
.chunks(usize::from(symbol_size))
.enumerate()
.map(|(esi, chunk)| {
let symbol = Symbol::new(
SymbolId::new(object_id, 0, u32::try_from(esi).expect("esi fits")),
chunk.to_vec(),
SymbolKind::Source,
);
crate::net::UdpInboundDatagram {
src_addr,
payload: encode_symbol_datagram(tag, 0, &symbol, None),
possibly_truncated: false,
}
})
.collect();
let batch = crate::net::UdpRecvBatch {
packets,
report: Default::default(),
};
let cx = Cx::for_testing();
let mut decoders = vec![decoder];
let stats = futures_lite::future::block_on(feed_datagram_batch_to_decoders(
&cx,
&batch,
tag,
false,
None,
&mut decoders,
symbol_size,
true,
))
.expect("feed plain source datagram batch");
let decoder = decoders.pop().expect("decoder");
assert_eq!(stats.observed, 4);
assert_eq!(stats.accepted, 4);
assert_eq!(stats.payload_bytes, 16);
assert!(decoder.complete);
assert_eq!(decoder.bytes_written, 16);
assert!(
decoder.staging_file.is_none(),
"completed batch must release cached staging descriptor"
);
assert!(
decoder.source_write_buffer.is_empty(),
"completed batch must flush the coalesced source buffer"
);
assert_eq!(
std::fs::read(staging_path).expect("read plain-source batch"),
data
);
}
#[test]
fn authenticated_source_datagram_batch_verifies_in_chunks_and_rejects_bad_tag() {
let ctx = SecurityContext::for_testing(0xA17E);
let tag = 0xA77E;
let object_id = entry_object_id("auth-source-batch-run", 0);
let symbol_size = 4u16;
let symbols = RQ_AUTH_VERIFY_TARGET_CHUNK_SYMBOLS * 2;
let data: Vec<u8> = (0..symbols * usize::from(symbol_size))
.map(|byte| u8::try_from((byte % 251) + 1).expect("bounded byte"))
.collect();
let dir = tempfile::tempdir().expect("tempdir");
let staging_path = dir.path().join("entry0");
let decoder = source_streaming_test_decoder(
object_id,
staging_path.clone(),
u64::try_from(data.len()).expect("test size fits"),
symbol_size,
);
let bad_esi = symbols / 2;
let bad_start = bad_esi * usize::from(symbol_size);
let bad_symbol = Symbol::new(
SymbolId::new(object_id, 0, u32::try_from(bad_esi).expect("esi fits")),
data[bad_start..bad_start + usize::from(symbol_size)].to_vec(),
SymbolKind::Source,
);
let good_bad_slot = ctx.sign_symbol(&bad_symbol);
let mut bad_tag = *good_bad_slot.tag().as_bytes();
bad_tag[0] ^= 0x80;
let datagrams = data
.chunks(usize::from(symbol_size))
.enumerate()
.map(|(esi, chunk)| {
signed_source_datagram(
&ctx,
object_id,
u32::try_from(esi).expect("esi fits"),
chunk.to_vec(),
(esi == bad_esi).then_some(AuthenticationTag::from_bytes(bad_tag)),
)
})
.collect();
let batch = udp_recv_batch(datagrams);
let pool = crate::runtime::blocking_pool::BlockingPool::new(4, 4);
let cx = Cx::new(
crate::types::RegionId::new_for_test(51, 1),
crate::types::TaskId::new_for_test(51, 0),
crate::types::Budget::INFINITE,
)
.with_blocking_pool_handle(Some(pool.handle()));
assert!(
rq_auth_verify_width_for_cx(&cx, symbols) > 1,
"test fixture must exercise chunked auth verification"
);
let mut decoders = vec![decoder];
let stats = futures_lite::future::block_on(feed_datagram_batch_to_decoders(
&cx,
&batch,
tag,
true,
Some(&ctx),
&mut decoders,
symbol_size,
false,
))
.expect("feed auth source datagram batch");
let decoder = decoders.pop().expect("decoder");
assert_eq!(stats.observed, u64::try_from(symbols).unwrap());
assert_eq!(stats.source_observed, u64::try_from(symbols).unwrap());
assert_eq!(stats.accepted, u64::try_from(symbols - 1).unwrap());
assert_eq!(stats.source_accepted, u64::try_from(symbols - 1).unwrap());
assert_eq!(stats.pipeline_feed_micros, 0);
assert!(
!decoder.complete,
"tampered source must leave block incomplete"
);
assert_eq!(decoder.bytes_written, 0);
assert_eq!(decoder.source_blocks[0].received_count, symbols - 1);
assert_eq!(decoder.source_blocks[0].auth_tags[bad_esi], None);
let staged = std::fs::read(staging_path).expect("read auth-source batch");
assert_eq!(
&staged[bad_start..bad_start + usize::from(symbol_size)],
&[0, 0, 0, 0]
);
assert_eq!(&staged[..bad_start], &data[..bad_start]);
assert_eq!(
&staged[bad_start + usize::from(symbol_size)..],
&data[bad_start + usize::from(symbol_size)..]
);
}
#[test]
fn source_streaming_round_boundary_flush_keeps_cached_staging_file_hot() {
let object_id = entry_object_id("source-stream-cache-flush", 0);
let symbol_size = 4u16;
let dir = tempfile::tempdir().expect("tempdir");
let staging_path = dir.path().join("entry0");
let mut decoder = source_streaming_test_decoder(
object_id,
staging_path,
ENTRY_STAGING_FILE_CACHE_MIN_BYTES,
symbol_size,
);
decoder.cache_staging_file = true;
let first = ParsedDatagram {
entry: 0,
sbn: 0,
esi: 0,
kind: SymbolKind::Source,
auth_tag: None,
payload_len: 4,
header_len: 0,
};
assert!(
futures_lite::future::block_on(persist_source_symbol(
&mut decoder,
&first,
&[1, 2, 3, 4],
symbol_size,
))
.expect("persist first cached source symbol")
);
assert!(decoder.staging_file.is_none());
assert_eq!(decoder.staging_unflushed_bytes, 0);
assert_eq!(decoder.source_write_buffer_offset, Some(0));
assert_eq!(decoder.source_write_buffer, vec![1, 2, 3, 4]);
futures_lite::future::block_on(flush_cached_entry_staging_files(std::slice::from_mut(
&mut decoder,
)))
.expect("round-boundary flush");
assert!(
decoder.staging_file.is_some(),
"round-boundary flush should not close the hot descriptor"
);
assert_eq!(decoder.staging_cursor, Some(4));
assert_eq!(decoder.staging_unflushed_bytes, 0);
assert!(decoder.source_write_buffer.is_empty());
assert_eq!(decoder.source_write_buffer_offset, None);
}
#[test]
fn source_streaming_small_entry_cache_closes_on_round_boundary() {
let object_id = entry_object_id("source-stream-small-cache-flush", 0);
let symbol_size = 4u16;
let dir = tempfile::tempdir().expect("tempdir");
let staging_path = dir.path().join("entry0");
let mut decoder =
source_streaming_test_decoder(object_id, staging_path.clone(), 8, symbol_size);
decoder.cache_staging_file = true;
let first = ParsedDatagram {
entry: 0,
sbn: 0,
esi: 0,
kind: SymbolKind::Source,
auth_tag: None,
payload_len: 4,
header_len: 0,
};
assert!(
futures_lite::future::block_on(persist_source_symbol(
&mut decoder,
&first,
&[1, 2, 3, 4],
symbol_size,
))
.expect("persist first cached source symbol")
);
assert!(decoder.staging_file.is_none());
assert_eq!(decoder.staging_cursor, None);
assert_eq!(decoder.staging_unflushed_bytes, 0);
assert_eq!(decoder.source_write_buffer_offset, Some(0));
assert_eq!(decoder.source_write_buffer, vec![1, 2, 3, 4]);
futures_lite::future::block_on(flush_cached_entry_staging_files(std::slice::from_mut(
&mut decoder,
)))
.expect("round-boundary flush");
assert!(
decoder.staging_file.is_none(),
"small-entry staging cache should not retain tree leaves across rounds"
);
assert_eq!(decoder.staging_cursor, None);
assert_eq!(decoder.staging_unflushed_bytes, 0);
assert!(decoder.source_write_buffer.is_empty());
assert_eq!(decoder.source_write_buffer_offset, None);
assert_eq!(
std::fs::read(staging_path).expect("read staged first source symbol"),
vec![1, 2, 3, 4, 0, 0, 0, 0]
);
}
#[test]
fn source_streaming_staging_cache_policy_batches_small_tree_entries() {
assert!(should_cache_entry_staging_file(
ENTRY_STAGING_FILE_CACHE_MIN_BYTES,
ENTRY_STAGING_FILE_CACHE_MAX_ENTRIES,
0,
));
assert!(!should_cache_entry_staging_file(
ENTRY_STAGING_FILE_CACHE_MIN_BYTES,
ENTRY_STAGING_FILE_CACHE_MAX_ENTRIES + 1,
0,
));
assert!(!should_cache_entry_staging_file(
ENTRY_STAGING_FILE_CACHE_MIN_BYTES - 1,
ENTRY_STAGING_FILE_CACHE_MAX_ENTRIES,
0,
));
assert!(should_cache_entry_staging_file(
ENTRY_STAGING_FILE_CACHE_MIN_BYTES - 1,
ENTRY_STAGING_FILE_CACHE_MAX_ENTRIES,
2,
));
assert!(!should_cache_entry_staging_file(
ENTRY_STAGING_FILE_CACHE_MIN_BYTES - 1,
ENTRY_STAGING_FILE_CACHE_MAX_ENTRIES + 1,
2,
));
assert!(!should_cache_entry_staging_file(
0,
ENTRY_STAGING_FILE_CACHE_MAX_ENTRIES,
2,
));
}
#[cfg(target_os = "linux")]
#[test]
fn e14_source_streaming_does_not_retain_one_staging_fd_per_entry() {
fn fd_count() -> usize {
std::fs::read_dir("/proc/self/fd")
.expect("read /proc/self/fd")
.count()
}
let dir = tempfile::tempdir().expect("tempdir");
let symbol_size = 1u16;
let parsed = ParsedDatagram {
entry: 0,
sbn: 0,
esi: 0,
kind: SymbolKind::Source,
auth_tag: None,
payload_len: 1,
header_len: 0,
};
let before = fd_count();
let mut decoders: Vec<EntryDecoder> = (0..1500)
.map(|idx| {
source_streaming_test_decoder(
entry_object_id("e14-fd-bound", u32::try_from(idx).expect("test index fits")),
dir.path().join(idx.to_string()),
1,
symbol_size,
)
})
.collect();
for (idx, decoder) in decoders.iter_mut().enumerate() {
let payload = [u8::try_from(idx % 251).expect("bounded byte")];
assert!(
futures_lite::future::block_on(persist_source_symbol(
decoder,
&parsed,
&payload,
symbol_size
))
.expect("persist source symbol"),
"entry {idx} source symbol should be accepted"
);
assert!(decoder.complete, "entry {idx} should complete");
assert!(decoder.staging_created, "entry {idx} should have staging");
assert_eq!(decoder.bytes_written, 1, "entry {idx} byte count");
}
let after = fd_count();
assert!(
after <= before + 64,
"receiver retained too many staging FDs: before={before} after={after}"
);
}
#[test]
fn signed_source_streaming_rejects_bad_tag_before_persist() {
let ctx = SecurityContext::for_testing(31338);
let object_id = entry_object_id("signed-source-stream-bad-tag", 0);
let symbol_size = 4u16;
let dir = tempfile::tempdir().expect("tempdir");
let staging_path = dir.path().join("entry0");
let mut decoder =
source_streaming_test_decoder(object_id, staging_path.clone(), 4, symbol_size);
let good = ctx.sign_symbol(&Symbol::new(
SymbolId::new(object_id, 0, 0),
vec![1, 2, 3, 4],
SymbolKind::Source,
));
let mut bad_tag = *good.tag().as_bytes();
bad_tag[0] ^= 0x80;
let (parsed, payload) = signed_source_payload(
&ctx,
object_id,
0,
vec![1, 2, 3, 4],
Some(AuthenticationTag::from_bytes(bad_tag)),
);
let accepted = futures_lite::future::block_on(feed_symbol(
&mut decoder,
&parsed,
&payload,
symbol_size,
Some(&ctx),
))
.expect("feed tampered source symbol");
assert!(!accepted);
assert!(!decoder.complete);
assert_eq!(decoder.bytes_written, 0);
assert!(!decoder.staging_created);
assert!(!staging_path.exists());
}
#[test]
fn signed_source_streaming_seeds_fec_decoder_from_staged_sources() {
let ctx = SecurityContext::for_testing(31339);
let object_id = entry_object_id("signed-source-stream-fec-seed", 0);
let symbol_size = 4u16;
let data = vec![1, 2, 3, 4, 5, 6, 7, 8];
let dir = tempfile::tempdir().expect("tempdir");
let staging_path = dir.path().join("entry0");
let mut decoder =
source_streaming_test_decoder(object_id, staging_path.clone(), 8, symbol_size);
let mut pipeline = DecodingPipeline::with_auth(
DecodingConfig {
symbol_size,
max_block_size: 8,
repair_overhead: 1.0,
min_overhead: 0,
max_buffered_symbols: 0,
block_timeout: std::time::Duration::from_secs(0),
verify_auth: true,
},
ctx.clone(),
);
pipeline
.set_object_params(object_params_for(object_id, 8, symbol_size, 8))
.expect("set object params");
decoder.pipeline = Some(pipeline);
let (first, first_payload) =
signed_source_payload(&ctx, object_id, 0, data[..4].to_vec(), None);
assert!(
futures_lite::future::block_on(feed_symbol(
&mut decoder,
&first,
&first_payload,
symbol_size,
Some(&ctx),
))
.expect("feed first source")
);
assert!(!decoder.complete);
assert_eq!(decoder.source_blocks[0].received_count, 1);
assert!(!decoder.source_blocks[0].pipeline_seeded[0]);
let pool = SymbolPool::new(PoolConfig::default());
let mut encoder = EncodingPipeline::new(
crate::config::EncodingConfig {
repair_overhead: 1.0,
max_block_size: 8,
symbol_size,
encoding_parallelism: 1,
decoding_parallelism: 1,
},
pool,
);
for encoded in encoder.encode_single_block_repair_range(object_id, 0, &data, 0, 4) {
let sym = encoded.expect("repair encode").into_symbol();
let auth = ctx.sign_symbol(&sym);
let dg = encode_symbol_datagram(0xA77E, 0, &sym, Some(auth.tag()));
let parsed = parse_symbol_header(&dg, 0xA77E, true).expect("parse signed repair");
let payload = dg[parsed.header_len..parsed.header_len + parsed.payload_len].to_vec();
let _ = futures_lite::future::block_on(feed_symbol(
&mut decoder,
&parsed,
&payload,
symbol_size,
Some(&ctx),
))
.expect("feed repair");
if decoder.complete {
break;
}
}
assert!(decoder.complete, "repair fallback should finish the block");
assert!(decoder.source_blocks[0].pipeline_seeded[0]);
assert_eq!(
std::fs::read(staging_path).expect("read repaired source stream"),
data
);
}
/// c54to7 regression (MATRIX-207): the FEC seed read-back must use the
/// SHARED-staging absolute offset (`staging_write_offset + block.start`),
/// not the entry-relative offset. A sharded large object (E-12) staged
/// every shard in one fragment; seeding a non-first shard at the raw
/// relative offset read shard 0's bytes, poisoning the seeded source
/// symbols — InconsistentEquations when redundancy caught it, a silent
/// rank-K-exact wrong solve (per-entry SHA mismatch at verify) when it
/// did not. Entry 0 (base 0) was immune, which produced the observed
/// entry-skew.
#[test]
fn signed_source_streaming_seed_reads_shard_absolute_staging_offset() {
let ctx = SecurityContext::for_testing(31341);
let object_id = entry_object_id("signed-source-stream-shard-seed", 1);
let symbol_size = 4u16;
let data = vec![11, 22, 33, 44, 55, 66, 77, 88];
let shard_base = 8u64;
let dir = tempfile::tempdir().expect("tempdir");
let staging_path = dir.path().join("fragment0");
// Shared fragment: shard 0's region holds poison bytes; this decoder
// is the SECOND shard, staged at absolute [8, 16).
std::fs::write(&staging_path, [0xAA; 16]).expect("pre-create shared fragment");
let mut decoder =
source_streaming_test_decoder(object_id, staging_path.clone(), 8, symbol_size);
decoder.staging_write_offset = shard_base;
decoder.staging_file_len = 16;
decoder.staging_shared = true;
let mut pipeline = DecodingPipeline::with_auth(
DecodingConfig {
symbol_size,
max_block_size: 8,
repair_overhead: 1.0,
min_overhead: 0,
max_buffered_symbols: 0,
block_timeout: std::time::Duration::from_secs(0),
verify_auth: true,
},
ctx.clone(),
);
pipeline
.set_object_params(object_params_for(object_id, 8, symbol_size, 8))
.expect("set object params");
decoder.pipeline = Some(pipeline);
let (first, first_payload) =
signed_source_payload(&ctx, object_id, 0, data[..4].to_vec(), None);
assert!(
futures_lite::future::block_on(feed_symbol(
&mut decoder,
&first,
&first_payload,
symbol_size,
Some(&ctx),
))
.expect("feed first source")
);
assert!(!decoder.complete);
assert_eq!(decoder.source_blocks[0].received_count, 1);
let pool = SymbolPool::new(PoolConfig::default());
let mut encoder = EncodingPipeline::new(
crate::config::EncodingConfig {
repair_overhead: 1.0,
max_block_size: 8,
symbol_size,
encoding_parallelism: 1,
decoding_parallelism: 1,
},
pool,
);
for encoded in encoder.encode_single_block_repair_range(object_id, 0, &data, 0, 4) {
let sym = encoded.expect("repair encode").into_symbol();
let auth = ctx.sign_symbol(&sym);
let dg = encode_symbol_datagram(0xA77E, 0, &sym, Some(auth.tag()));
let parsed = parse_symbol_header(&dg, 0xA77E, true).expect("parse signed repair");
let payload = dg[parsed.header_len..parsed.header_len + parsed.payload_len].to_vec();
let _ = futures_lite::future::block_on(feed_symbol(
&mut decoder,
&parsed,
&payload,
symbol_size,
Some(&ctx),
))
.expect("feed repair");
if decoder.complete {
break;
}
}
assert!(
decoder.complete,
"shard seed must read the staged source symbol from its ABSOLUTE \
shared-fragment offset and finish the block"
);
let staged = std::fs::read(staging_path).expect("read shared fragment");
assert_eq!(
&staged[8..16],
&data[..],
"shard content must decode byte-identical from absolute-offset seeds"
);
assert_eq!(
&staged[..8],
&[0xAA; 8],
"shard 0's region must never be touched by shard 1's decoder"
);
}
#[test]
fn source_streaming_round_boundary_seeds_when_source_arrives_after_repair() {
let object_id = entry_object_id("source-stream-round-boundary-seed", 0);
let symbol_size = 4u16;
let data = vec![1, 2, 3, 4, 5, 6, 7, 8];
let dir = tempfile::tempdir().expect("tempdir");
let staging_path = dir.path().join("entry0");
let mut decoder =
source_streaming_test_decoder(object_id, staging_path.clone(), 8, symbol_size);
decoder.cache_staging_file = true;
let mut pipeline = DecodingPipeline::new(DecodingConfig {
symbol_size,
max_block_size: 8,
repair_overhead: 1.0,
min_overhead: 0,
max_buffered_symbols: 0,
block_timeout: std::time::Duration::from_secs(0),
verify_auth: false,
});
pipeline
.set_object_params(object_params_for(object_id, 8, symbol_size, 8))
.expect("set object params");
decoder.pipeline = Some(pipeline);
let pool = SymbolPool::new(PoolConfig::default());
let mut encoder = EncodingPipeline::new(
crate::config::EncodingConfig {
repair_overhead: 1.0,
max_block_size: 8,
symbol_size,
encoding_parallelism: 1,
decoding_parallelism: 1,
},
pool,
);
let repair = encoder
.encode_single_block_repair_range(object_id, 0, &data, 0, 1)
.next()
.expect("one repair")
.expect("repair encode")
.into_symbol();
let repair_parsed = ParsedDatagram {
entry: 0,
sbn: repair.sbn(),
esi: repair.esi(),
kind: repair.kind(),
auth_tag: None,
payload_len: repair.data().len(),
header_len: 0,
};
assert!(
futures_lite::future::block_on(feed_symbol(
&mut decoder,
&repair_parsed,
repair.data(),
symbol_size,
None,
))
.expect("feed repair first")
);
assert!(!decoder.complete);
assert_eq!(decoder.source_blocks[0].received_count, 0);
let source = ParsedDatagram {
entry: 0,
sbn: 0,
esi: 0,
kind: SymbolKind::Source,
auth_tag: None,
payload_len: 4,
header_len: 0,
};
assert!(
futures_lite::future::block_on(feed_symbol(
&mut decoder,
&source,
&data[..4],
symbol_size,
None,
))
.expect("feed source after repair")
);
assert!(
!decoder.complete,
"no later repair arrived to trigger seeding"
);
assert!(source_streaming_block_ready_to_seed(&decoder, 0));
assert!(decoder.staging_file.is_none());
assert_eq!(decoder.staging_unflushed_bytes, 0);
assert_eq!(decoder.source_write_buffer_offset, Some(0));
assert_eq!(decoder.source_write_buffer, data[..4]);
let cx = Cx::for_testing();
let mut decoders = vec![decoder];
let seed_stats = futures_lite::future::block_on(
flush_and_seed_source_streaming_round_boundary(&cx, &mut decoders, symbol_size, None),
)
.expect("round-boundary seed");
assert_eq!(seed_stats.seeded, 1);
futures_lite::future::block_on(join_all_pending_decodes(
&cx,
&mut decoders,
RQ_MAX_PENDING_DECODE_JOBS_PER_TRANSFER_HARD,
))
.expect("join boundary decode");
let decoder = decoders.pop().expect("decoder");
assert!(decoder.complete, "round boundary seed should finish block");
assert_eq!(decoder.bytes_written, 8);
assert_eq!(
std::fs::read(staging_path).expect("read round-boundary repaired stream"),
data
);
}
// E-9 regression: a block completed via FEC (persist_decoded_block) must not be counted a
// second time when a late source retransmit for the same block arrives. Pre-fix, FEC left
// source_blocks[sbn].complete=false, so the late source's received_count==k path added
// block.len to bytes_written AGAIN → bytes_written != size → verify_and_commit falsely rejected
// a BYTE-CORRECT transfer as "per-entry SHA-256 mismatch" (the bad-regime non-convergence).
#[test]
fn e9_single_block_fec_then_late_source_does_not_double_count() {
let ctx = SecurityContext::for_testing(54321);
let object_id = entry_object_id("e9-mixed-no-double-count", 0);
let symbol_size = 4u16;
let data = vec![10u8, 20, 30, 40, 50, 60, 70, 80]; // 8 bytes, k=2 @ symbol_size 4
let dir = tempfile::tempdir().expect("tempdir");
let staging_path = dir.path().join("entry0");
let mut decoder =
source_streaming_test_decoder(object_id, staging_path.clone(), 8, symbol_size);
// One real source symbol arrives (esi=0): block not yet complete.
let (p0, pl0) = signed_source_payload(&ctx, object_id, 0, data[..4].to_vec(), None);
assert!(
futures_lite::future::block_on(feed_symbol(
&mut decoder,
&p0,
&pl0,
symbol_size,
Some(&ctx),
))
.expect("feed source 0")
);
assert!(!decoder.complete);
assert_eq!(decoder.source_blocks[0].received_count, 1);
// FEC completes the block (decoder emits the full block).
futures_lite::future::block_on(persist_decoded_block(&mut decoder, 0, &data))
.expect("persist decoded block");
assert!(
decoder.complete,
"FEC completion finishes the single-block entry"
);
assert_eq!(
decoder.bytes_written, 8,
"block counted exactly once after FEC"
);
assert!(
decoder.source_blocks[0].complete,
"FEC must mark the source block complete (E-9)"
);
// A LATE source retransmit for esi=1 arrives AFTER FEC completion. Pre-fix this drove
// received_count to k and DOUBLE-counted bytes_written to 16. It must be ignored now.
let (p1, pl1) = signed_source_payload(&ctx, object_id, 1, data[4..].to_vec(), None);
let _ = futures_lite::future::block_on(feed_symbol(
&mut decoder,
&p1,
&pl1,
symbol_size,
Some(&ctx),
));
assert_eq!(
decoder.bytes_written, 8,
"late source must NOT double-count bytes_written (E-9)"
);
assert_eq!(std::fs::read(&staging_path).expect("read staged"), data);
}
// E-9 regression (multi-block MIXED completion — the realistic bad-regime case): block 0 via
// FEC, block 1 via source, plus a late source retransmit for the already-FEC'd block 0. The
// entry must complete with bytes_written == size (each block counted ONCE) and byte-identical
// content. This directly exercises the source_blocks[sbn].complete guard that protects the
// multi-block path (where dec.complete is still false after the first block, so feed_symbol's
// dec.complete short-circuit does not hide the double-count).
#[test]
fn e9_multiblock_mixed_completion_counts_each_block_once() {
let object_id = entry_object_id("e9-multiblock-mixed", 0);
let symbol_size = 4u16;
let data = vec![1u8, 2, 3, 4, 5, 6, 7, 8]; // 8 bytes; max_block_size 4 -> 2 blocks, k=1 each
let dir = tempfile::tempdir().expect("tempdir");
let staging_path = dir.path().join("entry0");
let mut decoder = EntryDecoder {
index: 0,
object_id,
size: 8,
pipeline: None,
complete: false,
staging_path: staging_path.clone(),
staging_write_offset: 0,
staging_file_len: 8,
staging_shared: false,
staging_created: false,
staging_file: None,
staging_cursor: None,
staging_unflushed_bytes: 0,
cache_staging_file: false,
bytes_written: 0,
max_block_size: 4,
source_streaming: true,
source_blocks: source_block_progress_for(8, 4, symbol_size).expect("two source blocks"),
pending_decodes: Vec::new(),
inc: None,
inc_digest: None,
source_write_buffer: Vec::new(),
source_write_buffer_offset: None,
};
assert_eq!(decoder.source_blocks.len(), 2);
// Block 0 completes via FEC.
futures_lite::future::block_on(persist_decoded_block(&mut decoder, 0, &data[0..4]))
.expect("persist decoded block 0");
assert!(decoder.source_blocks[0].complete);
assert!(!decoder.complete, "block 1 still pending");
assert_eq!(decoder.bytes_written, 4);
// A LATE source retransmit for the already-FEC'd block 0 must be ignored (no double count).
let late = ParsedDatagram {
entry: 0,
sbn: 0,
esi: 0,
kind: SymbolKind::Source,
auth_tag: None,
payload_len: 4,
header_len: 0,
};
let accepted = futures_lite::future::block_on(persist_source_symbol(
&mut decoder,
&late,
&data[0..4],
symbol_size,
))
.expect("late source");
assert!(
!accepted,
"late source for a completed block is ignored (E-9)"
);
assert_eq!(decoder.bytes_written, 4, "no double count for block 0");
// Block 1 completes via source.
let b1 = ParsedDatagram {
entry: 0,
sbn: 1,
esi: 0,
kind: SymbolKind::Source,
auth_tag: None,
payload_len: 4,
header_len: 0,
};
assert!(
futures_lite::future::block_on(persist_source_symbol(
&mut decoder,
&b1,
&data[4..8],
symbol_size
))
.expect("block 1 source")
);
assert!(decoder.complete, "all blocks complete -> entry complete");
assert_eq!(
decoder.bytes_written, 8,
"each block counted once; bytes_written == size"
);
assert_eq!(std::fs::read(&staging_path).expect("read staged"), data);
}
#[test]
fn signed_datagram_feed_reaches_k512_decode_threshold() {
let ctx = SecurityContext::for_testing(101);
let object_id = entry_object_id("wire-k512", 0);
let symbol_size = 1024u16;
let max_block_size = DEFAULT_MAX_BLOCK_SIZE;
let data: Vec<u8> = (0usize..512 * 1024)
.map(|i: usize| (i.wrapping_mul(1_103_515_245) >> 16) as u8)
.collect();
let pool = SymbolPool::new(PoolConfig::default());
let mut encoder = EncodingPipeline::new(
crate::config::EncodingConfig {
repair_overhead: DEFAULT_REPAIR_OVERHEAD,
max_block_size,
symbol_size,
encoding_parallelism: 1,
decoding_parallelism: 1,
},
pool,
);
let params = object_params_for(
object_id,
data.len() as u64,
symbol_size,
max_block_size as u64,
);
let mut decoder = DecodingPipeline::with_auth(
DecodingConfig {
symbol_size,
max_block_size,
repair_overhead: DEFAULT_REPAIR_OVERHEAD,
min_overhead: 0,
max_buffered_symbols: 0,
block_timeout: std::time::Duration::from_secs(0),
verify_auth: true,
},
ctx.clone(),
);
decoder
.set_object_params(params)
.expect("set object params");
for encoded in encoder.encode_with_repair(object_id, &data, 512) {
let sym = encoded.expect("encode").into_symbol();
if sym.kind().is_source() && sym.esi() < 33 {
continue;
}
let auth = ctx.sign_symbol(&sym);
let dg = encode_symbol_datagram(0xABCD, 0, &sym, Some(auth.tag()));
let parsed = parse_symbol_header(&dg, 0xABCD, true).expect("parse signed datagram");
let payload = &dg[parsed.header_len..parsed.header_len + parsed.payload_len];
let received = Symbol::new(
SymbolId::new(object_id, parsed.sbn, parsed.esi),
payload.to_vec(),
parsed.kind,
);
let result = decoder
.feed(AuthenticatedSymbol::from_parts(
received,
parsed.auth_tag.expect("auth tag"),
))
.expect("feed");
if matches!(result, SymbolAcceptResult::BlockComplete { .. }) {
break;
}
}
assert!(
decoder.is_complete(),
"wire-parsed K=512 symbols must decode"
);
let mut out = decoder.into_data().expect("decoded data");
out.truncate(data.len());
assert_eq!(out, data);
}
#[test]
fn signed_datagram_rejects_missing_tag() {
let sym = Symbol::new(
SymbolId::new(ObjectId::new(1, 2), 3, 7),
vec![9u8; 1024],
SymbolKind::Repair,
);
let dg = encode_symbol_datagram(0xABCD, 42, &sym, None);
assert!(parse_symbol_header(&dg, 0xABCD, true).is_none());
}
#[test]
fn default_config_requires_symbol_auth_or_trusted_mode() {
let err = RqConfig::default()
.symbol_auth_context()
.expect_err("default config must fail closed");
assert!(matches!(err, RqError::Authentication(_)));
assert!(
RqConfig::default()
.allow_unauthenticated_for_trusted_transport()
.symbol_auth_context()
.expect("explicit trusted mode")
.is_none()
);
assert!(
RqConfig::default()
.with_symbol_auth(SecurityContext::for_testing(7))
.symbol_auth_context()
.expect("explicit auth context")
.is_some()
);
}
#[test]
fn datagram_rejects_wrong_tag() {
let sym = Symbol::new(
SymbolId::new(ObjectId::new(1, 2), 0, 0),
vec![0u8; 8],
SymbolKind::Source,
);
let dg = encode_symbol_datagram(0x1111, 0, &sym, None);
assert!(parse_symbol_header(&dg, 0x2222, false).is_none());
}
#[test]
fn datagram_rejects_bad_magic() {
let mut dg = encode_symbol_datagram(
0x1111,
0,
&Symbol::new(
SymbolId::new(ObjectId::new(1, 2), 0, 0),
vec![0u8; 8],
SymbolKind::Source,
),
None,
);
dg[0] ^= 0xFF;
assert!(parse_symbol_header(&dg, 0x1111, false).is_none());
}
#[test]
fn entry_object_id_is_deterministic_and_index_sensitive() {
let a = entry_object_id("deadbeef", 0);
let b = entry_object_id("deadbeef", 0);
let c = entry_object_id("deadbeef", 1);
assert_eq!(a, b);
assert_ne!(a, c);
}
#[test]
fn source_symbol_count_has_floor_and_ceils() {
assert_eq!(source_symbol_count(0, 1024), 1);
assert_eq!(source_symbol_count(1, 1024), 1);
assert_eq!(source_symbol_count(1024, 1024), 1);
assert_eq!(source_symbol_count(1025, 1024), 2);
}
#[test]
fn source_symbol_request_rebuilds_exact_source_payload() {
let config = RqConfig {
symbol_size: 512,
max_block_size: 1024,
..RqConfig::default()
};
let bytes: Vec<u8> = (0..1500).map(|i| (i % 251) as u8).collect();
let dir = tempfile::tempdir().expect("tempdir");
let source_path = dir.path().join("source.bin");
std::fs::write(&source_path, &bytes).expect("write source");
let enc = EntryEncoder {
index: 7,
object_id: entry_object_id("source-request", 7),
abs_path: source_path,
source_offset: 0,
size: bytes.len(),
repair_cursors: Vec::new(),
};
let first_block_tail = futures_lite::future::block_on(source_symbol_for_request(
&enc,
SourceSymbolRequest {
entry: 7,
sbn: 0,
esi: 1,
},
&config,
))
.expect("source symbol");
assert!(first_block_tail.kind().is_source());
assert_eq!(first_block_tail.sbn(), 0);
assert_eq!(first_block_tail.esi(), 1);
assert_eq!(first_block_tail.data(), &bytes[512..1024]);
let final_block = futures_lite::future::block_on(source_symbol_for_request(
&enc,
SourceSymbolRequest {
entry: 7,
sbn: 1,
esi: 0,
},
&config,
))
.expect("final source symbol");
assert_eq!(&final_block.data()[..476], &bytes[1024..]);
assert!(final_block.data()[476..].iter().all(|byte| *byte == 0));
}
#[test]
fn default_repair_overhead_is_source_first() {
assert_eq!(
initial_repair_target_per_block(512, DEFAULT_REPAIR_OVERHEAD),
0
);
}
#[test]
fn source_first_initial_repair_target_is_zero() {
assert_eq!(initial_repair_target_per_block(512, 1.0), 0);
}
#[test]
fn source_first_feedback_repair_batch_has_lossy_straggler_cushion() {
assert_eq!(
repair_target_for_feedback_round(512, 0, 1.0),
SOURCE_FIRST_FEEDBACK_REPAIR_FLOOR_PER_BLOCK
);
assert_eq!(
repair_target_for_feedback_round(512, 7, 1.0),
7 + SOURCE_FIRST_FEEDBACK_REPAIR_FLOOR_PER_BLOCK
);
}
#[test]
fn feedback_repair_batch_is_rate_matched_and_capped() {
assert_eq!(repair_target_for_feedback_round(512, 16, 1.03), 32);
assert_eq!(repair_target_for_feedback_round(512, 256, 1.50), 384);
}
#[test]
fn source_retransmit_is_bounded_by_default_in_source_first_mode() {
let config = RqConfig {
repair_overhead: 1.0,
..RqConfig::default()
};
assert_eq!(
source_retransmit_request_limit(&config, 1),
Some(DEFAULT_MAX_SOURCE_RETRANSMIT_REQUESTS)
);
assert_eq!(
source_retransmit_request_limit(&config, DEFAULT_SOURCE_RETRANSMIT_ROUNDS),
Some(DEFAULT_MAX_SOURCE_RETRANSMIT_REQUESTS)
);
assert_eq!(
source_retransmit_request_limit(&config, DEFAULT_SOURCE_RETRANSMIT_ROUNDS + 1),
None
);
}
#[test]
fn source_retransmit_stays_source_first_below_round0_loss_threshold() {
let config = RqConfig {
repair_overhead: 1.0,
round0_loss_target: 0.001,
..RqConfig::default()
};
assert_eq!(
source_retransmit_request_limit(&config, 1),
Some(DEFAULT_MAX_SOURCE_RETRANSMIT_REQUESTS)
);
assert!(!source_retransmit_needs_fec_fallback(&config, 1, 1, 0.0));
assert!(
source_retransmit_needs_fec_fallback(&config, 1, 0, 0.0),
"pending rank-only repair feedback must not wait for the source retransmit window"
);
}
#[test]
fn round0_loss_target_uses_repair_feedback_in_lossy_cells() {
let config = RqConfig {
repair_overhead: 1.0,
round0_loss_target: 0.02,
..RqConfig::default()
};
// Loss-target cells keep sparse source requests available in EVERY
// feedback round: after the round-0 FEC bulk, the residual is a few
// rank-deficient blocks that only targeted systematic retransmits can
// repair efficiently (MATRIX-207). The FEC fallback stays latched for
// rounds whose request list saturates.
assert_eq!(
source_retransmit_request_limit(&config, 1),
Some(DEFAULT_MAX_SOURCE_RETRANSMIT_REQUESTS)
);
assert_eq!(
source_retransmit_request_limit(&config, DEFAULT_SOURCE_RETRANSMIT_ROUNDS + 5),
Some(DEFAULT_MAX_SOURCE_RETRANSMIT_REQUESTS)
);
assert!(source_retransmit_needs_fec_fallback(&config, 1, 0, 0.0));
}
#[test]
fn source_retransmit_requires_explicit_round_budget() {
let config = RqConfig {
repair_overhead: 1.0,
source_retransmit_rounds: 2,
max_source_retransmit_requests: 17,
..RqConfig::default()
};
assert_eq!(source_retransmit_request_limit(&config, 1), Some(17));
assert_eq!(source_retransmit_request_limit(&config, 2), Some(17));
assert_eq!(source_retransmit_request_limit(&config, 3), None);
}
#[test]
fn source_retransmit_does_not_override_proactive_repair_mode() {
let config = RqConfig {
repair_overhead: 1.001,
source_retransmit_rounds: 2,
max_source_retransmit_requests: 17,
..RqConfig::default()
};
assert_eq!(source_retransmit_request_limit(&config, 1), None);
}
#[test]
fn source_retransmit_falls_back_to_fec_when_repair_only_saturated_or_final_round() {
let config = RqConfig {
repair_overhead: 1.0,
source_retransmit_rounds: 2,
max_source_retransmit_requests: 17,
..RqConfig::default()
};
assert!(source_retransmit_needs_fec_fallback(&config, 1, 0, 0.0));
assert!(!source_retransmit_needs_fec_fallback(&config, 1, 16, 0.0));
assert!(source_retransmit_needs_fec_fallback(&config, 1, 17, 0.0));
assert!(source_retransmit_needs_fec_fallback(&config, 2, 1, 0.0));
assert!(source_retransmit_needs_fec_fallback(&config, 2, 0, 0.0));
assert!(source_retransmit_needs_fec_fallback(&config, 3, 0, 0.0));
}
#[test]
fn source_retransmit_fec_fallback_uses_measured_loss_with_source_requests() {
let config = RqConfig {
repair_overhead: 1.0,
source_retransmit_rounds: 2,
max_source_retransmit_requests: 8192,
..RqConfig::default()
};
assert!(
!source_retransmit_needs_fec_fallback(&config, 1, 128, 0.001),
"near-clean feedback should preserve source-first repair"
);
assert!(
source_retransmit_needs_fec_fallback(&config, 1, 128, 0.10),
"broken-link measured loss must start calibrated FEC repair even while source requests remain"
);
}
#[test]
fn source_retransmit_repair_only_rounds_keep_fec_fallback_enabled() {
let config = RqConfig {
repair_overhead: 1.0,
source_retransmit_rounds: 2,
max_source_retransmit_requests: 17,
max_feedback_rounds: 16,
..RqConfig::default()
};
assert_eq!(source_retransmit_request_limit(&config, 3), None);
for feedback_round in (config.source_retransmit_rounds + 1)..=config.max_feedback_rounds {
assert!(
source_retransmit_needs_fec_fallback(&config, feedback_round, 0, 0.0),
"repair-only feedback round {feedback_round} must keep FEC fallback enabled"
);
}
}
#[test]
fn source_retransmit_fec_fallback_latches_after_repair_only_feedback() {
let config = RqConfig {
repair_overhead: 1.0,
source_retransmit_rounds: 2,
max_source_retransmit_requests: 17,
max_feedback_rounds: 16,
..RqConfig::default()
};
let mut active = false;
for (feedback_round, requested_sources, expected_active) in [
(1, 1, false),
(1, 0, true),
(2, 8, true),
(3, 1, true),
(config.max_feedback_rounds, 0, true),
] {
active |= source_retransmit_needs_fec_fallback(
&config,
feedback_round,
requested_sources,
0.0,
);
assert_eq!(
active, expected_active,
"feedback_round={feedback_round} requested_sources={requested_sources}"
);
}
}
#[test]
fn round0_loss_target_keeps_clean_and_good_links_source_first() {
for target in [0.0, 0.001] {
let config = RqConfig {
symbol_size: 1200,
max_block_size: 512 * 1024,
repair_overhead: 1.0,
round0_loss_target: target,
..RqConfig::default()
};
assert_eq!(round0_loss_target_repair_overhead(&config), 1.0);
assert_eq!(
source_retransmit_request_limit(&config, 1),
Some(DEFAULT_MAX_SOURCE_RETRANSMIT_REQUESTS)
);
assert_eq!(round0_bad_link_pacing_bps(&config), None);
}
}
#[test]
fn small_clean_round0_forces_source_only_repair_budget() {
let config = RqConfig {
symbol_size: 1200,
max_block_size: 512 * 1024,
repair_overhead: 1.001,
round0_loss_target: 0.0,
..RqConfig::default()
};
let tuning = RqRoundTuning {
repair_overhead: 1.08,
pacing: RqSprayPacing::from_rate(
RQ_MIN_PACING_BPS,
config.symbol_size,
RQ_ADAPTIVE_BURST_SYMBOLS,
None,
false,
),
};
let adjusted = apply_small_clean_round0_source_only(50 * 1024 * 1024, &config, tuning);
assert!(small_clean_source_only_round0(50 * 1024 * 1024, &config));
assert_eq!(adjusted.repair_overhead, 1.0);
assert_eq!(
initial_repair_target_per_block(437, adjusted.repair_overhead),
0
);
assert_eq!(
adjusted.pacing.rate_bytes_per_sec(),
RQ_COLD_START_PACING_BPS
);
assert!(round0_clean_ramp_enabled(&config, adjusted.pacing));
let mut pacer = RqSprayPacer::new_round0(
adjusted.pacing,
&config,
small_clean_source_only_round0(50 * 1024 * 1024, &config),
);
assert!(pacer.round0_ramp.is_some());
assert!(
pacer.small_clean_burst.is_some(),
"small clean UDP source-only sprays must use coarse burst pacing so sub-ms timer \
wakes cannot stretch 50M/perfect auth to the 60s timeout floor"
);
let burst_symbols = adjusted
.pacing
.max_burst_size
.max(u32::try_from(RQ_SEND_BATCH_PER_SOCKET).unwrap_or(u32::MAX));
let burst_bytes =
u64::from(adjusted.pacing.datagram_bytes).saturating_mul(u64::from(burst_symbols));
assert!(
duration_for_rate_window(burst_bytes, adjusted.pacing.rate_bytes_per_sec())
> RQ_PACING_MIN_PAUSE,
"small-clean burst pacing should sleep once per UDP batch, not once per symbol"
);
assert!(control_source_stream_eligible(50 * 1024 * 1024, &config));
let fallback_udp_pacer = RqSprayPacer::new_round0(
adjusted.pacing,
&config,
small_clean_source_only_round0(50 * 1024 * 1024, &config),
);
assert!(
fallback_udp_pacer.round0_ramp.is_some(),
"clean RQ fallback must still take the UDP clean-ramp path when control-source \
streaming is unavailable"
);
pacer.configure_with_shared_decision(
RqSprayPacing::from_rate(
RQ_COLD_START_PACING_BPS / 2,
config.symbol_size,
RQ_ADAPTIVE_BURST_SYMBOLS,
Some(Duration::from_millis(200)),
true,
),
None,
);
assert!(
pacer.small_clean_burst.is_none(),
"feedback/retry rounds must return to the normal per-datagram controller"
);
let large_clean_pacer = RqSprayPacer::new_round0(
RqSprayPacing::cold_start(config.symbol_size),
&config,
false,
);
assert!(large_clean_pacer.round0_ramp.is_some());
assert!(
large_clean_pacer.small_clean_burst.is_none(),
"large clean transfers keep the existing clean ramp without the small-transfer burst shim"
);
}
#[test]
fn matrix145_authenticated_clean_control_source_stream_is_eligible() {
let config = RqConfig {
symbol_size: 1200,
max_block_size: 512 * 1024,
repair_overhead: RQ_SMALL_CLEAN_SOURCE_ONLY_MAX_REPAIR_OVERHEAD,
round0_loss_target: 0.0,
max_transfer_bytes: 1024 * 1024 * 1024,
..RqConfig::default()
}
.with_symbol_auth(SecurityContext::for_testing(0xA7_50));
let total_bytes = 500 * 1024 * 1024;
let low_seed = RqSprayPacing::from_rate(
RQ_COLD_START_PACING_BPS / 4,
config.symbol_size,
RQ_ADAPTIVE_BURST_SYMBOLS,
None,
false,
);
let symbol_auth_enabled = config
.symbol_auth_context()
.expect("auth config is valid")
.is_some();
assert!(!small_clean_source_only_round0(total_bytes, &config));
assert!(symbol_auth_enabled);
assert!(control_source_stream_eligible(total_bytes, &config));
assert!(
round0_clean_ramp_enabled(&config, low_seed),
"MATRIX-145 fallback: clean authenticated UDP round 0 must not stay pinned at a low adaptive seed"
);
let pacer = RqSprayPacer::new_round0(low_seed, &config, false);
assert!(pacer.round0_ramp.is_some());
assert!(
pacer.small_clean_burst.is_none(),
"authenticated UDP fallback remains on the normal symbol path; only pacing should ramp"
);
}
#[test]
fn small_clean_round0_preserves_lossy_near_clean_debug_and_large_budgets() {
let lossy = RqConfig {
symbol_size: 1200,
max_block_size: 512 * 1024,
repair_overhead: 1.0,
round0_loss_target: 0.02,
..RqConfig::default()
};
let good = RqConfig {
symbol_size: 1200,
max_block_size: 512 * 1024,
repair_overhead: 1.0,
round0_loss_target: 0.001,
..RqConfig::default()
};
let explicit = RqConfig {
symbol_size: 1200,
max_block_size: 512 * 1024,
repair_overhead: 1.05,
round0_loss_target: 0.0,
..RqConfig::default()
};
let debug_drop = RqConfig {
symbol_size: 1200,
max_block_size: 512 * 1024,
repair_overhead: 1.05,
round0_loss_target: 0.0,
debug_drop_one_in: 17,
..RqConfig::default()
};
let large_clean = RqConfig {
symbol_size: 1200,
max_block_size: 512 * 1024,
repair_overhead: 1.0,
round0_loss_target: 0.0,
..RqConfig::default()
};
let tuning = RqRoundTuning {
repair_overhead: 1.08,
pacing: RqSprayPacing::from_rate(
RQ_MIN_PACING_BPS,
lossy.symbol_size,
RQ_ADAPTIVE_BURST_SYMBOLS,
None,
false,
),
};
assert!(!small_clean_source_only_round0(50 * 1024 * 1024, &lossy));
let lossy_adjusted = apply_small_clean_round0_source_only(50 * 1024 * 1024, &lossy, tuning);
assert_eq!(lossy_adjusted.repair_overhead, tuning.repair_overhead);
assert_eq!(
lossy_adjusted.pacing.rate_bytes_per_sec(),
tuning.pacing.rate_bytes_per_sec()
);
assert!(!small_clean_source_only_round0(50 * 1024 * 1024, &good));
let good_adjusted = apply_small_clean_round0_source_only(50 * 1024 * 1024, &good, tuning);
assert_eq!(good_adjusted.repair_overhead, tuning.repair_overhead);
assert_eq!(
good_adjusted.pacing.rate_bytes_per_sec(),
tuning.pacing.rate_bytes_per_sec()
);
assert!(!small_clean_source_only_round0(50 * 1024 * 1024, &explicit));
let explicit_adjusted =
apply_small_clean_round0_source_only(50 * 1024 * 1024, &explicit, tuning);
assert_eq!(explicit_adjusted.repair_overhead, tuning.repair_overhead);
assert_eq!(
explicit_adjusted.pacing.rate_bytes_per_sec(),
tuning.pacing.rate_bytes_per_sec()
);
assert!(!small_clean_source_only_round0(
50 * 1024 * 1024,
&debug_drop
));
let debug_drop_adjusted =
apply_small_clean_round0_source_only(50 * 1024 * 1024, &debug_drop, tuning);
assert_eq!(debug_drop_adjusted.repair_overhead, tuning.repair_overhead);
assert_eq!(
debug_drop_adjusted.pacing.rate_bytes_per_sec(),
tuning.pacing.rate_bytes_per_sec()
);
assert!(!small_clean_source_only_round0(
RQ_SMALL_CLEAN_SOURCE_ONLY_MAX_BYTES + 1,
&large_clean
));
let large_adjusted = apply_small_clean_round0_source_only(
RQ_SMALL_CLEAN_SOURCE_ONLY_MAX_BYTES + 1,
&large_clean,
tuning,
);
assert_eq!(large_adjusted.repair_overhead, tuning.repair_overhead);
assert_eq!(
large_adjusted.pacing.rate_bytes_per_sec(),
tuning.pacing.rate_bytes_per_sec()
);
}
#[test]
fn forced_round0_clean_ramp_cannot_override_lossy_config() {
for (name, round0_loss_target) in [("good", 0.001), ("bad", 0.02), ("broken", 0.10)] {
let config = RqConfig {
symbol_size: 1200,
max_block_size: 512 * 1024,
repair_overhead: 1.0,
round0_loss_target,
..RqConfig::default()
};
let pacing = RqSprayPacing::cold_start(config.symbol_size);
assert!(
!round0_clean_ramp_enabled(&config, pacing),
"{name} must not satisfy the clean-ramp predicate"
);
let pacer = RqSprayPacer::new_round0(pacing, &config, true);
assert!(
pacer.round0_ramp.is_none(),
"{name} must not force-enable the clean ramp on a lossy cell"
);
assert!(
pacer.small_clean_burst.is_none(),
"{name} must not force-enable the clean burst pacer on a lossy cell"
);
}
}
#[test]
fn control_source_stream_negotiates_for_clean_and_good_links_including_auth() {
let clean = RqConfig {
symbol_size: 1200,
max_block_size: 512 * 1024,
repair_overhead: 1.0,
round0_loss_target: 0.0,
..RqConfig::default()
};
let auth_clean = clean
.clone()
.with_symbol_auth(SecurityContext::for_testing(0xA7_51));
let good = RqConfig {
round0_loss_target: 0.001,
..clean.clone()
};
let auth_good = good
.clone()
.with_symbol_auth(SecurityContext::for_testing(0xA7_52));
let near_clean_above_good = RqConfig {
round0_loss_target: RQ_CONTROL_SOURCE_STREAM_MAX_LOSS_TARGET * 1.5,
..clean.clone()
};
let lossy = RqConfig {
round0_loss_target: 0.02,
..clean.clone()
};
let explicit_repair = RqConfig {
repair_overhead: 1.05,
..clean.clone()
};
let debug_drop = RqConfig {
debug_drop_one_in: 17,
..clean.clone()
};
let capped = RqConfig {
max_transfer_bytes: 128 * 1024 * 1024,
..clean.clone()
};
assert!(control_source_stream_eligible(50 * 1024 * 1024, &clean));
assert!(control_source_stream_eligible(500 * 1024 * 1024, &clean));
assert!(!small_clean_source_only_round0(500 * 1024 * 1024, &clean));
assert!(control_source_stream_eligible(
50 * 1024 * 1024,
&auth_clean
));
assert!(control_source_stream_eligible(50 * 1024 * 1024, &good));
assert!(control_source_stream_eligible(50 * 1024 * 1024, &auth_good));
assert!(!control_source_stream_eligible(
50 * 1024 * 1024,
&near_clean_above_good
));
assert!(!control_source_stream_eligible(50 * 1024 * 1024, &lossy));
assert!(!control_source_stream_eligible(
50 * 1024 * 1024,
&explicit_repair
));
assert!(!control_source_stream_eligible(
50 * 1024 * 1024,
&debug_drop
));
assert!(!control_source_stream_eligible(500 * 1024 * 1024, &capped));
}
#[test]
fn control_source_data_frame_roundtrips_entry_offset_and_payload() {
let frame = control_source_data_frame_with_auth(
"control-source-test",
7,
123_456,
b"payload",
None,
)
.expect("frame");
assert_eq!(frame.frame_type(), FrameType::ObjectData);
let parsed =
parse_control_source_data_frame(&frame, "control-source-test", None).expect("parse");
assert_eq!(parsed.entry, 7);
assert_eq!(parsed.offset, 123_456);
assert_eq!(parsed.data, b"payload");
let canonical = frame.to_wire_bytes().expect("canonical wire");
let direct =
control_source_data_wire_frame("control-source-test", 7, 123_456, b"payload", None)
.expect("direct wire");
assert_eq!(direct.as_ref(), canonical.as_slice());
assert!(
frame.encoded_len()
<= usize::try_from(crate::net::atp::protocol::frames::MAX_FRAME_SIZE).unwrap()
);
}
#[test]
fn control_source_data_chunk_stays_within_frame_cap() {
let payload = vec![0u8; RQ_CONTROL_SOURCE_CHUNK_BYTES];
let frame =
control_source_data_frame_with_auth("control-source-test", 7, 123_456, &payload, None)
.expect("frame");
let canonical = frame.to_wire_bytes().expect("canonical wire");
let direct =
control_source_data_wire_frame("control-source-test", 7, 123_456, &payload, None)
.expect("direct wire");
let max_frame_size =
usize::try_from(crate::net::atp::protocol::frames::MAX_FRAME_SIZE).unwrap();
assert_eq!(RQ_CONTROL_SOURCE_FRAME_MAX_BYTES, max_frame_size);
assert_eq!(frame.encoded_len(), max_frame_size);
assert_eq!(canonical.len(), max_frame_size);
assert_eq!(direct.as_ref(), canonical.as_slice());
let too_large = vec![0u8; RQ_CONTROL_SOURCE_CHUNK_BYTES + 1];
assert!(
control_source_data_wire_frame("control-source-test", 7, 123_456, &too_large, None)
.is_err()
);
}
#[test]
fn authenticated_control_source_data_rejects_tampered_payload() {
let context = SecurityContext::for_testing(0x145);
let frame = control_source_data_frame_with_auth(
"matrix-145",
7,
123_456,
b"payload",
Some(&context),
)
.expect("authenticated frame");
let parsed = parse_control_source_data_frame(&frame, "matrix-145", Some(&context))
.expect("authenticated parse");
assert_eq!(parsed.entry, 7);
assert_eq!(parsed.offset, 123_456);
assert_eq!(parsed.data, b"payload");
assert_eq!(
frame.payload().len(),
RQ_CONTROL_SOURCE_AUTH_DATA_HEADER + b"payload".len()
);
let mut tampered = frame.payload().to_vec();
let last = tampered.last_mut().expect("payload byte");
*last ^= 0x01;
let tampered_frame = Frame::new(ProtocolVersion::CURRENT, FrameType::ObjectData, tampered)
.expect("tampered frame");
assert!(matches!(
parse_control_source_data_frame(&tampered_frame, "matrix-145", Some(&context)),
Err(RqError::Authentication(_))
));
}
#[test]
fn authenticated_control_source_data_chunk_stays_within_frame_cap() {
let ctx = SecurityContext::for_testing(0xA7_52);
let payload = vec![0u8; RQ_CONTROL_SOURCE_AUTH_CHUNK_BYTES];
let frame = control_source_data_frame_with_auth(
"matrix145-auth-cap",
7,
123_456,
&payload,
Some(&ctx),
)
.expect("frame");
let canonical = frame.to_wire_bytes().expect("canonical wire");
let direct =
control_source_data_wire_frame("matrix145-auth-cap", 7, 123_456, &payload, Some(&ctx))
.expect("direct wire");
let max_frame_size =
usize::try_from(crate::net::atp::protocol::frames::MAX_FRAME_SIZE).unwrap();
assert_eq!(frame.encoded_len(), max_frame_size);
assert_eq!(canonical.len(), max_frame_size);
assert_eq!(direct.as_ref(), canonical.as_slice());
let too_large = vec![0u8; RQ_CONTROL_SOURCE_AUTH_CHUNK_BYTES + 1];
assert!(
control_source_data_wire_frame(
"matrix145-auth-cap",
7,
123_456,
&too_large,
Some(&ctx),
)
.is_err()
);
}
#[test]
fn authenticated_control_source_data_rejects_tampered_byte_before_write() {
let ctx = SecurityContext::for_testing(0xA7_53);
let transfer_id = "matrix145-auth-tamper";
let payload = b"payload".to_vec();
let frame = control_source_data_frame_with_auth(transfer_id, 0, 0, &payload, Some(&ctx))
.expect("signed frame");
let mut tampered_payload = frame.payload().to_vec();
*tampered_payload.last_mut().expect("payload byte") ^= 0x80;
let tampered = Frame::new(
ProtocolVersion::CURRENT,
FrameType::ObjectData,
tampered_payload,
)
.expect("tampered frame");
let dir = tempfile::tempdir().expect("tempdir");
let staging_path = dir.path().join("entry0");
let decoder = source_streaming_test_decoder(
entry_object_id(transfer_id, 0),
staging_path.clone(),
u64::try_from(payload.len()).expect("test payload length fits"),
4,
);
let mut decoders = vec![decoder];
let manifest = manifest_with(Vec::new(), 0);
let mut logical = std::collections::BTreeMap::new();
let mut logical_done = std::collections::BTreeMap::new();
let err = futures_lite::future::block_on(apply_control_source_data_frame(
&tampered,
transfer_id,
Some(&ctx),
&mut decoders,
&manifest,
&mut logical,
&mut logical_done,
))
.expect_err("tampered control-source byte must reject");
assert!(matches!(err, RqError::Authentication(_)));
assert_eq!(decoders[0].bytes_written, 0);
assert!(!decoders[0].complete);
assert!(
!staging_path.exists(),
"tampered authenticated control-source frame must not write staging bytes"
);
}
#[derive(Default)]
struct CountingControlIo {
bytes: Vec<u8>,
flushes: usize,
}
impl crate::io::AsyncRead for CountingControlIo {
fn poll_read(
self: std::pin::Pin<&mut Self>,
_cx: &mut std::task::Context<'_>,
_buf: &mut ReadBuf<'_>,
) -> std::task::Poll<std::io::Result<()>> {
std::task::Poll::Ready(Ok(()))
}
}
impl crate::io::AsyncWrite for CountingControlIo {
fn poll_write(
mut self: std::pin::Pin<&mut Self>,
_cx: &mut std::task::Context<'_>,
buf: &[u8],
) -> std::task::Poll<std::io::Result<usize>> {
self.bytes.extend_from_slice(buf);
std::task::Poll::Ready(Ok(buf.len()))
}
fn poll_flush(
mut self: std::pin::Pin<&mut Self>,
_cx: &mut std::task::Context<'_>,
) -> std::task::Poll<std::io::Result<()>> {
self.flushes += 1;
std::task::Poll::Ready(Ok(()))
}
fn poll_shutdown(
self: std::pin::Pin<&mut Self>,
_cx: &mut std::task::Context<'_>,
) -> std::task::Poll<std::io::Result<()>> {
std::task::Poll::Ready(Ok(()))
}
}
#[derive(Default)]
struct PendingCloseControlIo;
impl crate::io::AsyncRead for PendingCloseControlIo {
fn poll_read(
self: std::pin::Pin<&mut Self>,
_cx: &mut std::task::Context<'_>,
_buf: &mut ReadBuf<'_>,
) -> std::task::Poll<std::io::Result<()>> {
std::task::Poll::Pending
}
}
impl crate::io::AsyncWrite for PendingCloseControlIo {
fn poll_write(
self: std::pin::Pin<&mut Self>,
_cx: &mut std::task::Context<'_>,
buf: &[u8],
) -> std::task::Poll<std::io::Result<usize>> {
std::task::Poll::Ready(Ok(buf.len()))
}
fn poll_flush(
self: std::pin::Pin<&mut Self>,
_cx: &mut std::task::Context<'_>,
) -> std::task::Poll<std::io::Result<()>> {
std::task::Poll::Ready(Ok(()))
}
fn poll_shutdown(
self: std::pin::Pin<&mut Self>,
_cx: &mut std::task::Context<'_>,
) -> std::task::Poll<std::io::Result<()>> {
std::task::Poll::Ready(Ok(()))
}
}
#[test]
fn proof_close_drain_does_not_inherit_accept_timeout() {
let cx = Cx::for_testing();
let mut control = FrameTransport::new(PendingCloseControlIo);
let started = Instant::now();
futures_lite::future::block_on(drain_sender_close_after_proof(&cx, &mut control, "test"));
assert!(
started.elapsed() < Duration::from_secs(2),
"post-Proof close drain must not wait for the 60s accept/connect timeout"
);
}
#[test]
fn frame_transport_unflushed_send_defers_flush_until_requested() {
let frame = control_source_data_frame(7, 0, b"payload").expect("frame");
let mut control = FrameTransport::new(CountingControlIo::default());
let written = futures_lite::future::block_on(control.send_unflushed(&frame)).expect("send");
assert_eq!(control.stream.flushes, 0);
assert_eq!(control.stream.bytes.len(), written);
futures_lite::future::block_on(control.flush()).expect("flush");
assert_eq!(control.stream.flushes, 1);
}
#[test]
fn frame_transport_control_source_data_uses_canonical_wire_bytes_without_flush() {
let frame = control_source_data_frame(7, 123_456, b"payload").expect("frame");
let canonical = frame.to_wire_bytes().expect("canonical wire");
let mut control = FrameTransport::new(CountingControlIo::default());
let written = futures_lite::future::block_on(control.send_control_source_data_unflushed(
"control-source-test",
7,
123_456,
b"payload",
None,
))
.expect("send");
assert_eq!(written, canonical.len());
assert_eq!(control.stream.bytes, canonical);
assert_eq!(control.stream.flushes, 0);
}
#[test]
fn control_source_bulk_flush_batches_multiple_data_frames() {
let payload = vec![0u8; RQ_CONTROL_SOURCE_CHUNK_BYTES];
let frame = control_source_data_frame(7, 0, &payload).expect("frame");
let wire_len = frame.to_wire_bytes().expect("wire").len();
assert!(
RQ_CONTROL_SOURCE_FLUSH_BYTES >= wire_len * 8,
"bulk source stream should flush groups of data frames, not every frame"
);
assert!(RQ_CONTROL_SOURCE_FLUSH_BYTES <= 16 * 1024 * 1024);
}
#[test]
fn round0_bad_link_pacing_cap_is_narrow_to_bad_matrix_loss() {
for (target, expected) in [
(0.0, None),
(0.001, None),
(0.02, Some(RQ_BAD_LINK_ROUND0_PACING_BPS)),
(0.10, Some(RQ_BROKEN_LINK_ROUND0_PACING_BPS)),
] {
let config = RqConfig {
round0_loss_target: target,
..RqConfig::default()
};
assert_eq!(round0_bad_link_pacing_bps(&config), expected);
}
}
#[test]
fn round0_loss_target_calibrates_bad_link_repair_without_full_round_budget() {
let config = RqConfig {
symbol_size: 1200,
max_block_size: 512 * 1024,
repair_overhead: 1.0,
round0_loss_target: 0.02,
..RqConfig::default()
};
let mut state = RqAdaptiveSendState::new(7, &config, 8);
let tuning = state.round0_tuning(&config);
let extra_fraction = tuning.repair_overhead - 1.0;
let loss_bar = round0_loss_target_loss_bar(&config);
let full_round_budget = adaptive::overhead_for_target(
fixed_block_k(&config),
loss_bar,
RQ_SOURCE_FEC_FALLBACK_ALPHA,
RQ_SOURCE_FEC_FALLBACK_MAX_OVERHEAD,
);
assert!(
(1.04..=1.10).contains(&tuning.repair_overhead),
"2% target loss should produce a byte-efficient first-flight repair budget, got {}",
tuning.repair_overhead
);
assert!(
extra_fraction < full_round_budget,
"round-0 pre-spray should not spend the full feedback round budget: first_flight={extra_fraction} full_round={full_round_budget}"
);
assert!(
adaptive::decode_fail_probability(fixed_block_k(&config), extra_fraction, loss_bar)
<= RQ_ROUND0_TARGET_ALPHA * 1.000_001,
"round-0 pre-spray must still hit the round-0 decode-failure target \
(α={RQ_ROUND0_TARGET_ALPHA}; feedback rounds carry the residual, MATRIX-207)"
);
assert!(
initial_repair_target_per_block(437, tuning.repair_overhead) >= 20,
"2% loss should pre-spray enough repair symbols to cover the decode-failure target"
);
assert!(
tuning.pacing.path_rate_bps
<= RqSprayPacing::cold_start(config.symbol_size).path_rate_bps,
"round-0 loss calibration must not raise sender pacing"
);
assert_eq!(
tuning.pacing.rate_bytes_per_sec(),
RQ_BAD_LINK_ROUND0_PACING_BPS,
"2% bad-link round 0 should pace near the 50 mbit pipe instead of cold-start overrun"
);
assert!(
source_retransmit_needs_fec_fallback(&config, 2, 0, 0.0),
"repair-only rounds at the source-retransmit boundary must keep FEC fallback enabled"
);
}
#[test]
fn large_lossy_round0_uses_bounded_parallel_encode_plan() {
let matrix_50m = 50_u64 * 1024 * 1024;
let large = 500_u64 * 1024 * 1024;
let clean = RqConfig {
round0_loss_target: 0.0,
..RqConfig::default()
};
assert_eq!(parallel_encode_plan_for_transfer(large, &clean), None);
let good = RqConfig {
round0_loss_target: RQ_ROUND0_TARGET_LOSS_ENABLE_MIN / 5.0,
..RqConfig::default()
};
assert_eq!(parallel_encode_plan_for_transfer(large, &good), None);
let bad = RqConfig {
round0_loss_target: 0.02,
..RqConfig::default()
};
assert_eq!(
parallel_encode_plan_for_transfer(matrix_50m, &bad),
Some(ParallelEncodePlan {
max_batch_blocks: LOSSY_LARGE_PARALLEL_ENCODE_BATCH_BLOCKS
}),
"50M bad-link matrix cells must use the bounded lossy encode window to cap sender RSS"
);
assert_eq!(
parallel_encode_plan_for_transfer(large, &bad),
Some(ParallelEncodePlan {
max_batch_blocks: LOSSY_LARGE_PARALLEL_ENCODE_BATCH_BLOCKS
})
);
let broken = RqConfig {
round0_loss_target: 0.10,
..RqConfig::default()
};
assert_eq!(
parallel_encode_plan_for_transfer(matrix_50m, &broken),
Some(ParallelEncodePlan {
max_batch_blocks: LOSSY_LARGE_PARALLEL_ENCODE_BATCH_BLOCKS
}),
"50M broken-link matrix cells must use the bounded lossy encode window to cap sender RSS"
);
assert!(should_parallel_encode_source_blocks(
MAX_RAPTORQ_SOURCE_BLOCKS,
parallel_encode_plan_for_transfer(large, &bad)
));
assert!(!should_parallel_encode_source_blocks(
MAX_RAPTORQ_SOURCE_BLOCKS + 1,
parallel_encode_plan_for_transfer(large, &bad)
));
}
#[test]
fn small_transfer_keeps_existing_full_parallel_encode_plan() {
let small = PARALLEL_ENCODE_MAX_BYTES;
let config = RqConfig::default();
assert_eq!(
parallel_encode_plan_for_transfer(small, &config),
Some(ParallelEncodePlan {
max_batch_blocks: PARALLEL_ENCODE_HOST_MAX_BATCH_BLOCKS
})
);
}
#[test]
fn round0_loss_target_for_broken_link_is_bounded() {
let config = RqConfig {
symbol_size: 1200,
max_block_size: 512 * 1024,
repair_overhead: 1.0,
round0_loss_target: 0.10,
..RqConfig::default()
};
let overhead = round0_loss_target_repair_overhead(&config);
let mut state = RqAdaptiveSendState::new(7, &config, 1);
let tuning = state.round0_tuning(&config);
let loss_bar = round0_loss_target_loss_bar(&config);
assert!(
overhead > 1.0 + loss_bar,
"broken link must receive proactive FEC above the expected-loss floor: got {overhead}, floor {}",
1.0 + loss_bar
);
assert!(
overhead < 1.25,
"round-0 first flight must stay byte-efficient (α={RQ_ROUND0_TARGET_ALPHA} \
relaxation, MATRIX-207): the old 1e-6 per-block target inflated round-0 \
to +25.3% and lost the 500M/broken cell on wire time alone; got {overhead}"
);
assert_eq!(
tuning.pacing.rate_bytes_per_sec(),
RQ_BROKEN_LINK_ROUND0_PACING_BPS,
"10% broken-link round 0 must pace near the shaped 10 mbit pipe instead of cold-start overrun"
);
}
#[test]
fn source_retransmit_fec_fallback_uses_adaptive_overhead() {
let config = RqConfig {
symbol_size: 1024,
max_block_size: 512 * 1024,
repair_overhead: 1.0,
..RqConfig::default()
};
let mut state = RqAdaptiveSendState::new(99, &config, 4);
state.loss_ema = 0.03;
state.loss_bar = 0.05;
state.pacing_loss_ema = 0.05;
state.est.loss_p_hat = 0.05;
let tuning = state.source_fec_fallback_tuning(&config);
let expected = adaptive::overhead_for_target(
fixed_block_k(&config),
0.05,
RQ_SOURCE_FEC_FALLBACK_ALPHA,
RQ_SOURCE_FEC_FALLBACK_MAX_OVERHEAD,
);
assert!(
tuning.repair_overhead >= 1.0 + expected,
"fallback must apply adaptive FEC overhead: got {}, expected at least {}",
tuning.repair_overhead,
1.0 + expected
);
}
#[test]
fn measured_feedback_repair_overhead_tracks_receiver_loss() {
assert_eq!(measured_feedback_repair_overhead(0.0), 0.0);
assert_eq!(measured_feedback_repair_overhead(0.001), 0.0);
assert_eq!(measured_feedback_repair_overhead(-0.10), 0.0);
assert_eq!(measured_feedback_repair_overhead(f64::NAN), 0.0);
assert_eq!(measured_feedback_repair_overhead(f64::INFINITY), 0.0);
let bad = measured_feedback_repair_overhead(0.02);
assert!(
(0.029..=0.031).contains(&bad),
"2% measured loss should request about 3% repair overhead, got {bad}"
);
let broken = measured_feedback_repair_overhead(0.10);
assert!(
(0.12..=0.15).contains(&broken),
"10% measured loss should request a bounded 12-15% repair overhead, got {broken}"
);
assert_eq!(
measured_feedback_repair_overhead(0.90),
RQ_SOURCE_FEC_FALLBACK_MAX_OVERHEAD,
"pathological measured loss must clamp at the repair-overhead budget"
);
}
#[test]
fn source_fec_fallback_uses_measured_loss_floor_for_feedback_rounds() {
let config = RqConfig {
symbol_size: 1024,
max_block_size: 512 * 1024,
repair_overhead: 1.0,
..RqConfig::default()
};
let mut state = RqAdaptiveSendState::new(103, &config, 4);
state.last_round_loss_fraction = 0.10;
state.loss_ema = 0.0;
state.loss_bar = 0.0;
state.pacing_loss_ema = 0.0;
state.est.loss_p_hat = 0.0;
let tuning = state.source_fec_fallback_tuning(&config);
let measured = measured_feedback_repair_overhead(0.10);
assert!(
tuning.repair_overhead >= 1.0 + measured,
"fallback must honor receiver-measured loss floor: got {}, expected at least {}",
tuning.repair_overhead,
1.0 + measured
);
assert!(
repair_target_for_feedback_round(512, 0, tuning.repair_overhead) >= 66,
"10% broken-link repair round should send enough fresh repair symbols to avoid serial RTT rounds"
);
}
#[test]
fn source_fec_fallback_preserves_configured_broken_loss_floor() {
let config = RqConfig {
symbol_size: 1024,
max_block_size: 512 * 1024,
repair_overhead: 1.0,
round0_loss_target: 0.10,
..RqConfig::default()
};
let mut state = RqAdaptiveSendState::new(104, &config, 4);
state.last_round_loss_fraction = 0.0;
state.loss_ema = 0.0;
state.loss_bar = 0.0;
state.pacing_loss_ema = 0.0;
state.est.loss_p_hat = 0.0;
let loss_bar = state.source_fec_fallback_loss_bar(&config);
let expected_loss_bar = round0_loss_target_loss_bar(&config);
let tuning = state.source_fec_fallback_tuning(&config);
let expected = adaptive::overhead_for_target(
fixed_block_k(&config),
expected_loss_bar,
RQ_SOURCE_FEC_FALLBACK_ALPHA,
RQ_SOURCE_FEC_FALLBACK_MAX_OVERHEAD,
);
assert_eq!(
loss_bar, expected_loss_bar,
"broken-regime repair must keep the configured loss floor even when receiver loss is underreported"
);
assert!(
tuning.repair_overhead >= 1.0 + expected,
"zero reported feedback loss must not shrink broken-link repair below the configured loss target"
);
assert!(
repair_target_for_feedback_round(512, 0, tuning.repair_overhead) >= 100,
"10% broken-link repair should not devolve into low single-digit repair rounds"
);
}
#[test]
fn measured_loss_repair_target_fills_deficit_without_whole_block_respray() {
let block_source_n = 512;
let measured = measured_feedback_repair_overhead(0.10);
let repair_overhead = 1.0 + measured;
let initial_target = initial_repair_target_per_block(block_source_n, repair_overhead);
assert!(
(62..=77).contains(&initial_target),
"10% measured loss should calibrate to roughly 12-15% repair, got {initial_target}"
);
assert_eq!(
repair_target_for_feedback_round(block_source_n, initial_target - 1, repair_overhead),
initial_target,
"feedback repair should fill only the remaining measured-loss deficit"
);
let follow_up_target =
repair_target_for_feedback_round(block_source_n, initial_target, repair_overhead);
assert_eq!(
follow_up_target,
initial_target
+ adaptive_feedback_repair_batch_per_block(block_source_n, repair_overhead),
"once the measured-loss target is satisfied, later rounds should add one bounded batch"
);
assert!(
follow_up_target < block_source_n / 3,
"measured-loss feedback must stay bounded and never respray the whole block"
);
}
#[test]
fn source_fec_fallback_preserves_clean_link_batching_floor() {
let config = RqConfig {
symbol_size: 1024,
max_block_size: 512 * 1024,
repair_overhead: 1.0,
..RqConfig::default()
};
let mut state = RqAdaptiveSendState::new(101, &config, 4);
state.loss_ema = 0.0;
state.loss_bar = 0.0;
state.pacing_loss_ema = 0.0;
state.est.loss_p_hat = 0.0;
let tuning = state.source_fec_fallback_tuning(&config);
assert_eq!(
state.source_fec_fallback_loss_bar(&config),
RQ_SOURCE_FEC_FALLBACK_MIN_LOSS_BAR
);
assert!(
tuning.repair_overhead >= 1.0 + RQ_SOURCE_FEC_FALLBACK_MIN_OVERHEAD,
"clean-link fallback must preserve batching floor after E-RESYNC-16: got {}",
tuning.repair_overhead
);
}
#[test]
fn source_fec_fallback_keeps_near_clean_batching_floor() {
let config = RqConfig {
symbol_size: 1024,
max_block_size: 512 * 1024,
repair_overhead: 1.0,
..RqConfig::default()
};
let mut state = RqAdaptiveSendState::new(102, &config, 4);
state.loss_ema = 0.001;
state.loss_bar = 0.001;
state.pacing_loss_ema = 0.001;
state.est.loss_p_hat = 0.001;
let tuning = state.source_fec_fallback_tuning(&config);
let expected = adaptive::overhead_for_target(
fixed_block_k(&config),
RQ_SOURCE_FEC_FALLBACK_MIN_LOSS_BAR,
RQ_SOURCE_FEC_FALLBACK_ALPHA,
RQ_SOURCE_FEC_FALLBACK_MAX_OVERHEAD,
);
assert_eq!(
state.source_fec_fallback_loss_bar(&config),
RQ_SOURCE_FEC_FALLBACK_MIN_LOSS_BAR
);
assert!(tuning.repair_overhead >= 1.0 + expected);
assert!(
tuning.repair_overhead >= 1.0 + RQ_SOURCE_FEC_FALLBACK_MIN_OVERHEAD,
"near-clean fallback should batch repair symbols, got {}",
tuning.repair_overhead
);
}
#[test]
fn proactive_initial_repair_target_ceilings_extra_fraction() {
assert_eq!(initial_repair_target_per_block(512, 1.15), 77);
assert_eq!(initial_repair_target_per_block(1, 1.01), 1);
}
#[test]
fn round0_loss_target_respects_explicit_repair_overhead_floor() {
let config = RqConfig {
repair_overhead: 1.25,
round0_loss_target: 0.02,
..RqConfig::default()
};
assert!(round0_loss_target_repair_overhead(&config) >= 1.25);
}
#[test]
fn effective_block_size_preserves_k512_streaming_target_for_normal_files() {
let config = RqConfig::default();
let symbol_size = usize::from(config.symbol_size);
assert_eq!(config.symbol_size, DEFAULT_SYMBOL_SIZE);
let expected_target = symbol_size
.saturating_mul(TARGET_SOURCE_SYMBOLS_PER_BLOCK)
.min(TARGET_STREAMING_BLOCK_BYTES)
.min(config.max_block_size);
assert_eq!(
expected_target.div_ceil(symbol_size),
TARGET_SOURCE_SYMBOLS_PER_BLOCK,
"default streaming target must stay at K512 even after symbol-size changes"
);
let effective = effective_max_block_size_for_largest_entry(&config, 10 * 1024 * 1024)
.expect("10MiB should fit");
assert_eq!(effective, expected_target);
assert_eq!(
max_block_source_symbol_count(10 * 1024 * 1024, config.symbol_size, effective),
TARGET_SOURCE_SYMBOLS_PER_BLOCK
);
}
#[test]
fn effective_block_size_grows_from_k512_only_to_fit_sbn_limit() {
let config = RqConfig::default();
let one_gib: usize = 1024 * 1024 * 1024;
let symbol_size = usize::from(config.symbol_size);
let streaming_target = symbol_size
.saturating_mul(TARGET_SOURCE_SYMBOLS_PER_BLOCK)
.min(TARGET_STREAMING_BLOCK_BYTES);
assert_eq!(
streaming_target.div_ceil(symbol_size),
TARGET_SOURCE_SYMBOLS_PER_BLOCK,
"fixture starts from the default K512 streaming target"
);
let min_symbol_aligned_block = one_gib
.div_ceil(MAX_SOURCE_BLOCKS)
.max(symbol_size)
.div_ceil(symbol_size)
.saturating_mul(symbol_size);
let effective = effective_max_block_size_for_largest_entry(&config, one_gib)
.expect("1GiB should fit default transfer geometry");
assert_eq!(
effective, min_symbol_aligned_block,
"large entries should grow only enough to fit the u8 SBN limit"
);
assert!(
effective > streaming_target,
"this fixture must exercise SBN-limit growth beyond the normal streaming target"
);
assert!(
one_gib.div_ceil(effective - symbol_size) > MAX_SOURCE_BLOCKS,
"one symbol-aligned step smaller should exceed the SBN wire limit"
);
assert_eq!(one_gib.div_ceil(effective), MAX_SOURCE_BLOCKS);
}
#[test]
fn effective_block_size_rejects_unsplit_huge_entries() {
// E-12: a 5 GiB logical file must be split into multiple bounded
// RaptorQ objects before this helper runs. If an unsplit object reaches
// this point, fail closed instead of growing K above the configured max.
let config = RqConfig::default();
let symbol_size = usize::from(config.symbol_size);
let five_gib: usize = 5 * 1024 * 1024 * 1024;
assert!(matches!(
effective_max_block_size_for_largest_entry(&config, five_gib),
Err(RqError::Coding(msg)) if msg.starts_with("[ASUP-E803]")
));
// Entries within the configured 2 GiB default ceiling are unaffected (byte-identical).
let one_gib: usize = 1024 * 1024 * 1024;
assert_eq!(
effective_max_block_size_for_largest_entry(&config, one_gib).unwrap(),
one_gib
.div_ceil(MAX_SOURCE_BLOCKS)
.max(symbol_size)
.div_ceil(symbol_size)
.saturating_mul(symbol_size)
);
// One byte beyond the configured object ceiling fails closed unless it
// has first been split into multiple objects.
let ceiling = config.max_block_size.saturating_mul(MAX_SOURCE_BLOCKS);
assert!(matches!(
effective_max_block_size_for_largest_entry(&config, ceiling + 1),
Err(RqError::Coding(msg)) if msg.starts_with("[ASUP-E803]")
));
}
#[test]
fn max_block_source_symbols_uses_effective_block_not_entry_size() {
let config = RqConfig::default();
let symbol_size = usize::from(config.symbol_size);
let effective_k512_block = symbol_size * TARGET_SOURCE_SYMBOLS_PER_BLOCK;
assert_eq!(
max_block_source_symbol_count(
10 * 1024 * 1024,
config.symbol_size,
effective_k512_block
),
TARGET_SOURCE_SYMBOLS_PER_BLOCK
);
assert_eq!(
max_block_source_symbol_count(
10 * 1024 * 1024,
config.symbol_size,
DEFAULT_MAX_BLOCK_SIZE
),
DEFAULT_MAX_BLOCK_SIZE.div_ceil(symbol_size)
);
}
fn m1_test_encoding_config(config: &RqConfig) -> crate::config::EncodingConfig {
crate::config::EncodingConfig {
repair_overhead: config.repair_overhead,
max_block_size: config.max_block_size,
symbol_size: config.symbol_size,
encoding_parallelism: 1,
decoding_parallelism: 1,
}
}
fn symbol_fingerprint(symbol: &Symbol) -> (u8, u32, SymbolKind, Vec<u8>) {
(
symbol.id().sbn(),
symbol.id().esi(),
symbol.kind(),
symbol.data().to_vec(),
)
}
fn collect_monolithic_symbols(
object_id: ObjectId,
bytes: &[u8],
config: &RqConfig,
repair_count: usize,
) -> Vec<(u8, u32, SymbolKind, Vec<u8>)> {
let mut pipeline = EncodingPipeline::new(
m1_test_encoding_config(config),
SymbolPool::new(PoolConfig::default()),
);
pipeline
.encode_with_repair(object_id, bytes, repair_count)
.map(|encoded| {
let encoded = encoded.expect("monolithic encode succeeds");
symbol_fingerprint(encoded.symbol())
})
.collect()
}
fn collect_m1_source_symbols(
object_id: ObjectId,
bytes: &[u8],
config: &RqConfig,
repair_count: usize,
) -> Vec<(u8, u32, SymbolKind, Vec<u8>)> {
let mut symbols = Vec::new();
for block in encode_ahead_blocks(bytes.len(), config).expect("block plan") {
let mut pipeline = EncodingPipeline::new(
m1_test_encoding_config(config),
SymbolPool::new(PoolConfig::default()),
);
for encoded in pipeline.encode_single_block_with_repair(
object_id,
block.sbn,
&bytes[block.start..block.start + block.len],
repair_count,
) {
let encoded = encoded.expect("M=1 source encode succeeds");
symbols.push(symbol_fingerprint(encoded.symbol()));
}
}
symbols
}
fn collect_source_only_block_symbols(
object_id: ObjectId,
bytes: &[u8],
config: &RqConfig,
) -> Vec<(u8, u32, SymbolKind, Vec<u8>)> {
let mut symbols = Vec::new();
let symbol_size = usize::from(config.symbol_size);
for block in encode_ahead_blocks(bytes.len(), config).expect("block plan") {
for esi in 0..block.k {
let symbol = source_symbol_from_block(
object_id,
block.sbn,
esi,
&bytes[block.start..block.start + block.len],
symbol_size,
)
.expect("source-only symbol");
symbols.push(symbol_fingerprint(&symbol));
}
}
symbols
}
fn bonded_test_descriptor(bytes: &[u8], config: &RqConfig) -> BondTransferDescriptor {
let entry = ManifestEntry {
index: 0,
rel_path: "payload.bin".to_string(),
size: bytes.len() as u64,
sha256_hex: hex_encode(&Sha256::digest(bytes)),
members: Vec::new(),
fragment: None,
};
let manifest = TransferManifest {
transfer_id: "bonded-donor-spray-test".to_string(),
root_name: "payload".to_string(),
is_directory: false,
total_bytes: bytes.len() as u64,
merkle_root_hex: "00".repeat(32),
entries: vec![entry],
};
BondTransferDescriptor::from_manifest(
&manifest,
config.symbol_size,
config.max_block_size as u64,
None,
)
}
fn bonded_test_assignment(donor_index: u32, donor_count: u32) -> DonorAssignment {
DonorAssignment::new_static(
donor_index,
donor_count,
vec![std::net::SocketAddr::from(([127, 0, 0, 1], 48123))],
None,
)
}
fn collect_bonded_donor_test_symbols(
descriptor: &BondTransferDescriptor,
assignment: &DonorAssignment,
bytes: &[u8],
config: &RqConfig,
) -> Vec<(u8, u32, SymbolKind, Vec<u8>)> {
let repair_symbols_per_block =
bonded_initial_repair_symbols_per_block(config).expect("repair budget");
let schedule =
schedule_bonded_donor_spray(descriptor, assignment, repair_symbols_per_block)
.expect("bonded schedule");
let mut symbols = Vec::new();
for block in &schedule.blocks {
let start = usize::try_from(block.geometry.block_start).expect("test block start");
let len = usize::try_from(block.geometry.block_bytes).expect("test block len");
for emission in block.symbol_emissions(schedule.donor_index) {
let symbol =
encode_bonded_donor_emission(emission, &bytes[start..start + len], config)
.expect("bonded emission encodes");
symbols.push(symbol_fingerprint(&symbol));
}
}
symbols
}
#[test]
fn bonded_donor_count_one_matches_single_source_round0_symbols() {
let config = RqConfig {
symbol_size: 4,
max_block_size: 8,
repair_overhead: 2.0,
..RqConfig::default()
};
let bytes = b"abcdefghijklmnop";
let descriptor = bonded_test_descriptor(bytes, &config);
let assignment = bonded_test_assignment(0, 1);
let object_id = descriptor.entry_object_id(0);
let actual = collect_bonded_donor_test_symbols(&descriptor, &assignment, bytes, &config);
let expected = collect_m1_source_symbols(object_id, bytes, &config, 2);
assert_eq!(actual, expected);
}
#[test]
fn bonded_donor_symbols_stay_inside_static_residue_class() {
let config = RqConfig {
symbol_size: 4,
max_block_size: 8,
repair_overhead: 2.0,
..RqConfig::default()
};
let bytes = b"abcdefghijklmnop";
let descriptor = bonded_test_descriptor(bytes, &config);
let assignment = bonded_test_assignment(1, 2);
let symbols = collect_bonded_donor_test_symbols(&descriptor, &assignment, bytes, &config);
assert!(
symbols
.iter()
.any(|(_, _, kind, _)| *kind == SymbolKind::Source),
"donor should keep the receiver's source-first path alive"
);
assert!(
symbols
.iter()
.any(|(_, _, kind, _)| *kind == SymbolKind::Repair),
"donor should emit assigned repair symbols too"
);
for (_, esi, _, _) in symbols {
assert_eq!(esi % 2, 1, "donor emitted out-of-residue ESI {esi}");
}
}
#[test]
fn bonded_donor_round0_pacing_decision_is_reported_and_bounded() {
let config = RqConfig {
symbol_size: 1200,
max_block_size: 512 * 1024,
repair_overhead: 1.0,
round0_loss_target: 0.0,
..RqConfig::default()
};
let decision = bonded_donor_round0_pacing_decision("bonded-donor-pacing", &config, 2);
let pacer = RqSprayPacer::new_round0(decision.pacing, &config, false);
assert_eq!(
decision.report.initial_rate_bytes_per_sec,
decision.pacing.rate_bytes_per_sec()
);
assert_eq!(
decision.report.final_rate_bytes_per_sec,
decision.report.initial_rate_bytes_per_sec
);
assert_eq!(
decision.report.burst_symbols,
decision.pacing.max_burst_size
);
assert_eq!(decision.report.burst_bytes, decision.pacing.burst_bytes());
assert_eq!(
decision.report.datagram_bytes,
decision.pacing.datagram_bytes
);
assert!(
decision.report.burst_bytes < decision.report.initial_rate_bytes_per_sec,
"donor pacing must bound a send burst below a full second of path budget"
);
assert_eq!(
decision.report.clean_round0_ramp_enabled,
pacer.round0_ramp.is_some()
);
}
fn collect_monolithic_repair_symbols(
object_id: ObjectId,
bytes: &[u8],
config: &RqConfig,
first_repair: usize,
repair_count: usize,
) -> Vec<(u8, u32, SymbolKind, Vec<u8>)> {
let mut pipeline = EncodingPipeline::new(
m1_test_encoding_config(config),
SymbolPool::new(PoolConfig::default()),
);
pipeline
.encode_repair_range(object_id, bytes, first_repair, repair_count)
.map(|encoded| {
let encoded = encoded.expect("monolithic repair encode succeeds");
symbol_fingerprint(encoded.symbol())
})
.collect()
}
fn collect_m1_repair_symbols(
object_id: ObjectId,
bytes: &[u8],
config: &RqConfig,
first_repair: usize,
repair_count: usize,
) -> Vec<(u8, u32, SymbolKind, Vec<u8>)> {
let mut symbols = Vec::new();
for block in encode_ahead_blocks(bytes.len(), config).expect("block plan") {
let mut pipeline = EncodingPipeline::new(
m1_test_encoding_config(config),
SymbolPool::new(PoolConfig::default()),
);
for encoded in pipeline.encode_single_block_repair_range(
object_id,
block.sbn,
&bytes[block.start..block.start + block.len],
first_repair,
repair_count,
) {
let encoded = encoded.expect("M=1 repair encode succeeds");
symbols.push(symbol_fingerprint(encoded.symbol()));
}
}
symbols
}
#[test]
fn encode_ahead_ring_is_single_slot_fifo() {
let mut ring = EncodeAheadRing::default();
assert_eq!(EncodeAheadRing::CAPACITY, 1);
let object_id = ObjectId::new_for_test(0xF204);
let first = EncodeAheadSymbol {
entry: 7,
symbol: Symbol::new(
SymbolId::new(object_id, 0, 0),
vec![1, 2, 3],
SymbolKind::Source,
),
};
let second = EncodeAheadSymbol {
entry: 8,
symbol: Symbol::new(
SymbolId::new(object_id, 0, 1),
vec![4, 5, 6],
SymbolKind::Source,
),
};
ring.push(first).expect("first symbol fits");
assert!(matches!(
ring.push(second),
Err(RqError::Coding(message)) if message.contains("ring is full")
));
let popped = ring.pop().expect("first symbol queued");
assert_eq!(popped.entry, 7);
assert_eq!(popped.symbol.id().esi(), 0);
assert!(ring.is_empty());
}
#[test]
fn encode_ahead_blocks_match_monolithic_block_geometry() {
let config = RqConfig {
symbol_size: 4,
max_block_size: 6,
..RqConfig::default()
};
assert_eq!(
encode_ahead_blocks(13, &config).expect("block plan"),
vec![
EncodeAheadBlock {
sbn: 0,
start: 0,
len: 6,
k: 2,
},
EncodeAheadBlock {
sbn: 1,
start: 6,
len: 6,
k: 2,
},
EncodeAheadBlock {
sbn: 2,
start: 12,
len: 1,
k: 1,
},
]
);
assert!(
encode_ahead_blocks(0, &config)
.expect("empty plan")
.is_empty()
);
let small_blocks = RqConfig {
symbol_size: 8,
max_block_size: 3,
..RqConfig::default()
};
assert_eq!(
encode_ahead_blocks(7, &small_blocks).expect("small block plan"),
vec![
EncodeAheadBlock {
sbn: 0,
start: 0,
len: 3,
k: 1,
},
EncodeAheadBlock {
sbn: 1,
start: 3,
len: 3,
k: 1,
},
EncodeAheadBlock {
sbn: 2,
start: 6,
len: 1,
k: 1,
},
]
);
}
#[test]
fn read_source_range_reassembles_original_bytes_on_demand() {
let dir = tempfile::tempdir().expect("temp dir");
let path = dir.path().join("payload.bin");
let bytes: Vec<u8> = (0..257).map(|i| (i % 251) as u8).collect();
std::fs::write(&path, &bytes).expect("write payload");
let mut reassembled = Vec::new();
for (offset, len) in [(0, 17), (17, 64), (81, 128), (209, 48)] {
let chunk = futures_lite::future::block_on(read_source_range(&path, offset, len))
.expect("read source chunk");
reassembled.extend_from_slice(&chunk);
}
assert_eq!(reassembled, bytes);
}
#[test]
fn read_source_range_fails_closed_on_truncated_source() {
let dir = tempfile::tempdir().expect("temp dir");
let path = dir.path().join("payload.bin");
std::fs::write(&path, b"short").expect("write payload");
let err = futures_lite::future::block_on(read_source_range(&path, 2, 8))
.expect_err("range past EOF must fail");
assert!(
matches!(&err, RqError::Source(message) if message.contains("payload.bin")),
"expected source-path error, got {err:?}"
);
}
#[test]
fn source_block_progress_covers_complete_entries_or_disables_streaming() {
let progress = source_block_progress_for(5, 2, 2).expect("complete block table");
assert_eq!(progress.len(), 3);
assert_eq!(
progress
.iter()
.map(|block| (block.start, block.len, block.k, block.received.len()))
.collect::<Vec<_>>(),
vec![(0, 2, 1, 1), (2, 2, 1, 1), (4, 1, 1, 1)]
);
let too_many_blocks = u64::try_from(MAX_SOURCE_BLOCKS + 1).unwrap_or(u64::MAX);
assert!(
source_block_progress_for(too_many_blocks, 1, 1).is_none(),
"source streaming must fall back to the decoder when the SBN envelope is incomplete"
);
}
#[test]
fn m1_encode_ahead_source_and_initial_repair_is_byte_identical() {
let config = RqConfig {
symbol_size: 4,
max_block_size: 6,
repair_overhead: 1.0,
..RqConfig::default()
};
let object_id = ObjectId::new_for_test(0xF202);
let bytes = b"abcdefghijklmnopq";
assert_eq!(
collect_m1_source_symbols(object_id, bytes, &config, 2),
collect_monolithic_symbols(object_id, bytes, &config, 2)
);
}
#[test]
fn source_only_block_symbols_match_pipeline_source_symbols() {
let config = RqConfig {
symbol_size: 4,
max_block_size: 6,
repair_overhead: 1.0,
..RqConfig::default()
};
let object_id = ObjectId::new_for_test(0xF205);
let bytes = b"source-only-fast-path";
assert_eq!(
collect_source_only_block_symbols(object_id, bytes, &config),
collect_m1_source_symbols(object_id, bytes, &config, 0)
);
}
#[test]
fn m1_encode_ahead_repair_range_is_byte_identical() {
let config = RqConfig {
symbol_size: 4,
max_block_size: 6,
repair_overhead: 1.0,
..RqConfig::default()
};
let object_id = ObjectId::new_for_test(0xF203);
let bytes = b"repair-rounds-span-blocks";
assert_eq!(
collect_m1_repair_symbols(object_id, bytes, &config, 1, 3),
collect_monolithic_repair_symbols(object_id, bytes, &config, 1, 3)
);
}
#[test]
fn object_params_match_block_plan() {
// 3 MiB with 8 MiB blocks => 1 block; 1024-byte symbols => K=3072.
let p = object_params_for(ObjectId::new(0, 0), 3 * 1024 * 1024, 1024, 8 * 1024 * 1024);
assert_eq!(p.source_blocks, 1);
assert_eq!(p.symbols_per_block, 3072);
// 20 MiB with 8 MiB blocks => 3 blocks (8+8+4).
let p2 = object_params_for(ObjectId::new(0, 0), 20 * 1024 * 1024, 1024, 8 * 1024 * 1024);
assert_eq!(p2.source_blocks, 3);
assert_eq!(p2.symbols_per_block, 8192);
}
// ─── E-12 large-entry multi-object split ───────────────────────────────
fn digest_for_bytes(rel_path: &str, bytes: &[u8]) -> EntryDigest {
EntryDigest {
rel_path: rel_path.to_string(),
size: bytes.len() as u64,
content_id: crate::atp::object::ObjectId::content(
crate::atp::object::ContentId::from_bytes(bytes),
),
content_sha256: Sha256::digest(bytes).into(),
}
}
fn fragment_entry(
index: u32,
object_rel_path: &str,
object_bytes: &[u8],
fragment: LargeObjectFragment,
) -> ManifestEntry {
ManifestEntry {
index,
rel_path: object_rel_path.to_string(),
size: object_bytes.len() as u64,
sha256_hex: hex_encode(&Sha256::digest(object_bytes)),
members: Vec::new(),
fragment: Some(fragment),
}
}
fn two_fragment_manifest(a: &[u8], b: &[u8]) -> TransferManifest {
let mut whole = Vec::with_capacity(a.len().saturating_add(b.len()));
whole.extend_from_slice(a);
whole.extend_from_slice(b);
let whole_sha = hex_encode(&Sha256::digest(&whole));
TransferManifest {
transfer_id: "rqtransfer1".to_string(),
root_name: "huge.bin".to_string(),
is_directory: false,
total_bytes: whole.len() as u64,
merkle_root_hex: flat_merkle_root_from_digests(&[digest_for_bytes("huge.bin", &whole)]),
entries: vec![
fragment_entry(
0,
".atp-fragment-0-0",
a,
LargeObjectFragment {
rel_path: "huge.bin".to_string(),
shard_index: 0,
shard_count: 2,
logical_offset: 0,
len: a.len() as u64,
logical_size: whole.len() as u64,
sha256_hex: whole_sha.clone(),
},
),
fragment_entry(
1,
".atp-fragment-0-1",
b,
LargeObjectFragment {
rel_path: "huge.bin".to_string(),
shard_index: 1,
shard_count: 2,
logical_offset: a.len() as u64,
len: b.len() as u64,
logical_size: whole.len() as u64,
sha256_hex: whole_sha,
},
),
],
}
}
fn shared_fragment_decoders(
manifest: &TransferManifest,
staging_path: &Path,
complete: bool,
staging_created: bool,
) -> Vec<EntryDecoder> {
manifest
.entries
.iter()
.map(|entry| {
let fragment = entry.fragment.as_ref().expect("fragment metadata");
EntryDecoder {
index: entry.index,
object_id: entry_object_id(&manifest.transfer_id, entry.index),
size: entry.size,
pipeline: None,
complete,
staging_path: staging_path.to_path_buf(),
staging_write_offset: fragment.logical_offset,
staging_file_len: fragment.logical_size,
staging_shared: true,
staging_created,
staging_file: None,
staging_cursor: None,
staging_unflushed_bytes: 0,
cache_staging_file: false,
bytes_written: if complete { entry.size } else { 0 },
max_block_size: DEFAULT_MAX_BLOCK_SIZE,
source_streaming: false,
source_blocks: Vec::new(),
pending_decodes: Vec::new(),
inc: None,
inc_digest: None,
source_write_buffer: Vec::new(),
source_write_buffer_offset: None,
}
})
.collect()
}
#[cfg(unix)]
fn inode_for(path: &Path) -> u64 {
use std::os::unix::fs::MetadataExt;
std::fs::metadata(path).expect("metadata").ino()
}
#[test]
fn split_large_entries_plans_bounded_ranged_objects() {
let dir = tempfile::tempdir().expect("tempdir");
let bytes: Vec<u8> = (0..600).map(|i| (i % 251) as u8).collect();
let entry = source_entry(dir.path(), "huge.bin", &bytes);
let logical = vec![digest_for_bytes("huge.bin", &bytes)];
let config = RqConfig {
symbol_size: 1,
max_block_size: 1,
..RqConfig::default()
};
let split =
futures_lite::future::block_on(split_large_entries(vec![entry], &logical, &config))
.expect("split large entry");
assert_eq!(split.len(), 3, "600 bytes at 256-byte objects => 3 shards");
assert_eq!(split[0].source_offset, 0);
assert_eq!(split[0].source_len, Some(256));
assert_eq!(split[1].source_offset, 256);
assert_eq!(split[1].source_len, Some(256));
assert_eq!(split[2].source_offset, 512);
assert_eq!(split[2].source_len, Some(88));
for (idx, shard) in split.iter().enumerate() {
let fragment = shard.fragment.as_ref().expect("fragment metadata");
assert_eq!(fragment.rel_path, "huge.bin");
assert_eq!(fragment.shard_index, idx as u32);
assert_eq!(fragment.shard_count, 3);
assert_eq!(fragment.logical_size, bytes.len() as u64);
assert_eq!(fragment.sha256_hex, hex_encode(&Sha256::digest(&bytes)));
}
let mut buf = vec![0u8; 64];
let (range_size, _, range_sha) =
futures_lite::future::block_on(hash_source_entry_streaming(&split[1], &mut buf))
.expect("range hash");
assert_eq!(range_size, 256);
let expected_range_sha: [u8; 32] = Sha256::digest(&bytes[256..512]).into();
assert_eq!(range_sha, expected_range_sha);
}
#[test]
fn validate_manifest_accepts_and_bounds_fragment_table() {
let whole = b"abcdefghijklmnopqrstuvwxyz".to_vec();
let a = &whole[..10];
let b = &whole[10..];
let whole_sha = hex_encode(&Sha256::digest(&whole));
let entries = vec![
fragment_entry(
0,
".atp-fragment-0-0",
a,
LargeObjectFragment {
rel_path: "huge.bin".to_string(),
shard_index: 0,
shard_count: 2,
logical_offset: 0,
len: a.len() as u64,
logical_size: whole.len() as u64,
sha256_hex: whole_sha.clone(),
},
),
fragment_entry(
1,
".atp-fragment-0-1",
b,
LargeObjectFragment {
rel_path: "huge.bin".to_string(),
shard_index: 1,
shard_count: 2,
logical_offset: a.len() as u64,
len: b.len() as u64,
logical_size: whole.len() as u64,
sha256_hex: whole_sha,
},
),
];
let manifest = TransferManifest {
transfer_id: "rqtransfer1".to_string(),
root_name: "huge.bin".to_string(),
is_directory: false,
total_bytes: whole.len() as u64,
merkle_root_hex: flat_merkle_root_from_digests(&[digest_for_bytes("huge.bin", &whole)]),
entries,
};
assert!(validate_manifest(&manifest, &RqConfig::default()).is_ok());
let mut gapped = manifest.clone();
gapped.entries[0]
.fragment
.as_mut()
.expect("fragment")
.logical_size += 1;
gapped.entries[1]
.fragment
.as_mut()
.expect("fragment")
.logical_offset += 1;
gapped.entries[1]
.fragment
.as_mut()
.expect("fragment")
.logical_size += 1;
assert!(matches!(
validate_manifest(&gapped, &RqConfig::default()),
Err(RqError::Frame(m)) if m.contains("not contiguous")
));
}
#[test]
fn verify_and_commit_reassembles_fragmented_file() {
let dest = tempfile::tempdir().expect("dest dir");
let staging_dir = dest.path().join(".atp-rq-fragment-staging");
std::fs::create_dir_all(&staging_dir).expect("staging dir");
let a = b"first fragment ".to_vec();
let b = b"second fragment".to_vec();
let mut whole = Vec::new();
whole.extend_from_slice(&a);
whole.extend_from_slice(&b);
let a_path = staging_dir.join("0");
let b_path = staging_dir.join("1");
std::fs::write(&a_path, &a).expect("write first shard");
std::fs::write(&b_path, &b).expect("write second shard");
let whole_sha = hex_encode(&Sha256::digest(&whole));
let manifest = TransferManifest {
transfer_id: "rqtransfer1".to_string(),
root_name: "huge.bin".to_string(),
is_directory: false,
total_bytes: whole.len() as u64,
merkle_root_hex: flat_merkle_root_from_digests(&[digest_for_bytes("huge.bin", &whole)]),
entries: vec![
fragment_entry(
0,
".atp-fragment-0-0",
&a,
LargeObjectFragment {
rel_path: "huge.bin".to_string(),
shard_index: 0,
shard_count: 2,
logical_offset: 0,
len: a.len() as u64,
logical_size: whole.len() as u64,
sha256_hex: whole_sha.clone(),
},
),
fragment_entry(
1,
".atp-fragment-0-1",
&b,
LargeObjectFragment {
rel_path: "huge.bin".to_string(),
shard_index: 1,
shard_count: 2,
logical_offset: a.len() as u64,
len: b.len() as u64,
logical_size: whole.len() as u64,
sha256_hex: whole_sha,
},
),
],
};
let mut decoders = vec![
EntryDecoder {
index: 0,
object_id: entry_object_id(&manifest.transfer_id, 0),
size: a.len() as u64,
pipeline: None,
complete: true,
staging_path: a_path,
staging_write_offset: 0,
staging_file_len: a.len() as u64,
staging_shared: false,
staging_created: true,
staging_file: None,
staging_cursor: None,
staging_unflushed_bytes: 0,
cache_staging_file: false,
bytes_written: a.len() as u64,
max_block_size: DEFAULT_MAX_BLOCK_SIZE,
source_streaming: false,
source_blocks: Vec::new(),
pending_decodes: Vec::new(),
inc: None,
inc_digest: None,
source_write_buffer: Vec::new(),
source_write_buffer_offset: None,
},
EntryDecoder {
index: 1,
object_id: entry_object_id(&manifest.transfer_id, 1),
size: b.len() as u64,
pipeline: None,
complete: true,
staging_path: b_path,
staging_write_offset: 0,
staging_file_len: b.len() as u64,
staging_shared: false,
staging_created: true,
staging_file: None,
staging_cursor: None,
staging_unflushed_bytes: 0,
cache_staging_file: false,
bytes_written: b.len() as u64,
max_block_size: DEFAULT_MAX_BLOCK_SIZE,
source_streaming: false,
source_blocks: Vec::new(),
pending_decodes: Vec::new(),
inc: None,
inc_digest: None,
source_write_buffer: Vec::new(),
source_write_buffer_offset: None,
},
];
let receipt = futures_lite::future::block_on(verify_and_commit(
&manifest,
&mut decoders,
dest.path(),
0,
0,
&std::collections::BTreeMap::new(),
&CompletionDigestIndex::default(),
))
.expect("verify fragmented file");
assert!(receipt.committed, "fragmented transfer must commit");
assert!(receipt.sha_ok);
assert!(receipt.merkle_ok);
assert_eq!(receipt.files, 1);
assert_eq!(std::fs::read(dest.path().join("huge.bin")).unwrap(), whole);
}
#[test]
fn verify_and_commit_renames_contiguous_single_file_fragment_staging() {
let dest = tempfile::tempdir().expect("dest dir");
let receive_staging_dir = dest.path().join(".atp-rq-staging-rqtransfer1-0");
let fragment_dir = receive_staging_dir.join(RQ_SINGLE_FILE_FRAGMENT_STAGING_DIR);
std::fs::create_dir_all(&fragment_dir).expect("fragment staging dir");
let a = vec![b'a'; 4096];
let b = vec![b'b'; 4096];
let mut whole = Vec::with_capacity(a.len() + b.len());
whole.extend_from_slice(&a);
whole.extend_from_slice(&b);
let staging_path = fragment_dir.join("0");
std::fs::write(&staging_path, &whole).expect("write contiguous fragment staging");
let manifest = two_fragment_manifest(&a, &b);
let planned_staging = single_file_fragment_staging_path(&manifest, &receive_staging_dir)
.expect("single-file fragment staging path");
assert_eq!(planned_staging, staging_path);
let mut decoders = shared_fragment_decoders(&manifest, &staging_path, true, true);
#[cfg(unix)]
let staging_inode = inode_for(&staging_path);
let receipt = futures_lite::future::block_on(verify_and_commit(
&manifest,
&mut decoders,
dest.path(),
0,
0,
&BTreeMap::new(),
&CompletionDigestIndex::default(),
))
.expect("verify contiguous fragmented file");
assert!(receipt.committed, "fragmented transfer must commit");
assert!(receipt.sha_ok);
assert!(receipt.merkle_ok);
let out_path = dest.path().join("huge.bin");
assert_eq!(
std::fs::read(&out_path).expect("read committed file"),
whole
);
assert!(
!staging_path.exists(),
"rename branch must move the contiguous staging file"
);
#[cfg(unix)]
assert_eq!(
inode_for(&out_path),
staging_inode,
"committed file should be the renamed staging inode"
);
}
#[test]
fn verify_and_commit_rejects_tampered_contiguous_fragment_and_cleans_staging() {
let dest = tempfile::tempdir().expect("dest dir");
let receive_staging_dir = dest.path().join(".atp-rq-staging-rqtransfer1-0");
let fragment_dir = receive_staging_dir.join(RQ_SINGLE_FILE_FRAGMENT_STAGING_DIR);
std::fs::create_dir_all(&fragment_dir).expect("fragment staging dir");
let a = b"first verified fragment".to_vec();
let b = b"second verified fragment".to_vec();
let mut tampered = Vec::with_capacity(a.len() + b.len());
tampered.extend_from_slice(&a);
tampered.extend_from_slice(&b);
tampered[a.len()] ^= 0x5a;
let staging_path = fragment_dir.join("0");
std::fs::write(&staging_path, &tampered).expect("write tampered staging");
let manifest = two_fragment_manifest(&a, &b);
let mut decoders = shared_fragment_decoders(&manifest, &staging_path, true, true);
let receipt = futures_lite::future::block_on(verify_and_commit(
&manifest,
&mut decoders,
dest.path(),
0,
0,
&BTreeMap::new(),
&CompletionDigestIndex::default(),
))
.expect("verify returns fail-closed receipt");
assert!(!receipt.committed, "tampered fragment must not commit");
assert!(!receipt.sha_ok);
assert!(!dest.path().join("huge.bin").exists());
assert!(
!staging_path.exists(),
"failed contiguous fragment verification must clean staging"
);
}
#[test]
fn contiguous_fragment_staging_accepts_out_of_order_datagram_writes() {
let dest = tempfile::tempdir().expect("dest dir");
let receive_staging_dir = dest.path().join(".atp-rq-staging-rqtransfer1-0");
let fragment_dir = receive_staging_dir.join(RQ_SINGLE_FILE_FRAGMENT_STAGING_DIR);
let staging_path = fragment_dir.join("0");
let a = b"first fragment arrives second".to_vec();
let b = b"second fragment arrives first".to_vec();
let mut whole = Vec::with_capacity(a.len() + b.len());
whole.extend_from_slice(&a);
whole.extend_from_slice(&b);
let manifest = two_fragment_manifest(&a, &b);
let mut decoders = shared_fragment_decoders(&manifest, &staging_path, false, false);
futures_lite::future::block_on(write_entry_staging_range(&mut decoders[1], 0, &b))
.expect("write second fragment first");
futures_lite::future::block_on(write_entry_staging_range(&mut decoders[0], 0, &a))
.expect("write first fragment second");
for decoder in &mut decoders {
decoder.complete = true;
decoder.bytes_written = decoder.size;
}
let receipt = futures_lite::future::block_on(verify_and_commit(
&manifest,
&mut decoders,
dest.path(),
0,
0,
&BTreeMap::new(),
&CompletionDigestIndex::default(),
))
.expect("verify out-of-order contiguous fragments");
assert!(receipt.committed);
assert!(receipt.sha_ok);
assert!(receipt.merkle_ok);
assert_eq!(
std::fs::read(dest.path().join("huge.bin")).expect("read committed file"),
whole
);
}
#[test]
fn verify_and_commit_uses_source_stream_trailer_digests_for_manifest_placeholders() {
let dest = tempfile::tempdir().expect("dest dir");
let staging_dir = dest.path().join(".atp-rq-trailer-staging");
std::fs::create_dir_all(&staging_dir).expect("staging dir");
let a = b"source-stream fragment one ".to_vec();
let b = b"source-stream fragment two".to_vec();
let mut whole = Vec::new();
whole.extend_from_slice(&a);
whole.extend_from_slice(&b);
let a_path = staging_dir.join("0");
let b_path = staging_dir.join("1");
std::fs::write(&a_path, &a).expect("write first staged shard");
std::fs::write(&b_path, &b).expect("write second staged shard");
let placeholder = sha256_hex_placeholder();
let manifest = TransferManifest {
transfer_id: "rqtransfer-trailer".to_string(),
root_name: "huge.bin".to_string(),
is_directory: false,
total_bytes: whole.len() as u64,
merkle_root_hex: placeholder.clone(),
entries: vec![
ManifestEntry {
index: 0,
rel_path: ".atp-fragment-0-0".to_string(),
size: a.len() as u64,
sha256_hex: placeholder.clone(),
members: Vec::new(),
fragment: Some(LargeObjectFragment {
rel_path: "huge.bin".to_string(),
shard_index: 0,
shard_count: 2,
logical_offset: 0,
len: a.len() as u64,
logical_size: whole.len() as u64,
sha256_hex: placeholder.clone(),
}),
},
ManifestEntry {
index: 1,
rel_path: ".atp-fragment-0-1".to_string(),
size: b.len() as u64,
sha256_hex: placeholder,
members: Vec::new(),
fragment: Some(LargeObjectFragment {
rel_path: "huge.bin".to_string(),
shard_index: 1,
shard_count: 2,
logical_offset: a.len() as u64,
len: b.len() as u64,
logical_size: whole.len() as u64,
sha256_hex: sha256_hex_placeholder(),
}),
},
],
};
let a_digest = digest_for_bytes(".atp-fragment-0-0", &a);
let b_digest = digest_for_bytes(".atp-fragment-0-1", &b);
let logical_digest = digest_for_bytes("huge.bin", &whole);
let merkle_root_hex = flat_merkle_root_from_digests(&[logical_digest.clone()]);
let complete = RqRoundComplete {
round_symbols_sent: 0,
entry_digests: vec![
ObjectCompleteEntryDigest {
index: 0,
size: a_digest.size,
sha256_hex: hex_encode(&a_digest.content_sha256),
},
ObjectCompleteEntryDigest {
index: 1,
size: b_digest.size,
sha256_hex: hex_encode(&b_digest.content_sha256),
},
],
logical_digests: vec![ObjectCompleteLogicalDigest {
rel_path: "huge.bin".to_string(),
size: logical_digest.size,
sha256_hex: hex_encode(&logical_digest.content_sha256),
}],
merkle_root_hex: Some(merkle_root_hex),
};
let completion_digests =
CompletionDigestIndex::from_round_complete(&complete, &manifest, true)
.expect("source-stream trailer digests are valid");
let mut decoders = vec![
EntryDecoder {
index: 0,
object_id: entry_object_id(&manifest.transfer_id, 0),
size: a.len() as u64,
pipeline: None,
complete: true,
staging_path: a_path,
staging_write_offset: 0,
staging_file_len: a.len() as u64,
staging_shared: false,
staging_created: true,
staging_file: None,
staging_cursor: None,
staging_unflushed_bytes: 0,
cache_staging_file: false,
bytes_written: a.len() as u64,
max_block_size: DEFAULT_MAX_BLOCK_SIZE,
source_streaming: true,
source_blocks: Vec::new(),
pending_decodes: Vec::new(),
inc: None,
inc_digest: Some((
a_digest.size,
a_digest.content_id.clone(),
a_digest.content_sha256,
)),
source_write_buffer: Vec::new(),
source_write_buffer_offset: None,
},
EntryDecoder {
index: 1,
object_id: entry_object_id(&manifest.transfer_id, 1),
size: b.len() as u64,
pipeline: None,
complete: true,
staging_path: b_path,
staging_write_offset: 0,
staging_file_len: b.len() as u64,
staging_shared: false,
staging_created: true,
staging_file: None,
staging_cursor: None,
staging_unflushed_bytes: 0,
cache_staging_file: false,
bytes_written: b.len() as u64,
max_block_size: DEFAULT_MAX_BLOCK_SIZE,
source_streaming: true,
source_blocks: Vec::new(),
pending_decodes: Vec::new(),
inc: None,
inc_digest: Some((
b_digest.size,
b_digest.content_id.clone(),
b_digest.content_sha256,
)),
source_write_buffer: Vec::new(),
source_write_buffer_offset: None,
},
];
let mut logical_precomputed = BTreeMap::new();
logical_precomputed.insert(
"huge.bin".to_string(),
(
logical_digest.size,
logical_digest.content_id,
logical_digest.content_sha256,
),
);
let receipt = futures_lite::future::block_on(verify_and_commit(
&manifest,
&mut decoders,
dest.path(),
0,
0,
&logical_precomputed,
&completion_digests,
))
.expect("verify source-stream trailer digests");
assert!(receipt.committed, "trailer digests must authorize commit");
assert!(receipt.sha_ok);
assert!(receipt.merkle_ok);
assert_eq!(std::fs::read(dest.path().join("huge.bin")).unwrap(), whole);
}
#[test]
fn verify_and_commit_rejects_bad_source_stream_trailer_digest_without_commit() {
let dest = tempfile::tempdir().expect("dest dir");
let staging_dir = dest.path().join(".atp-rq-trailer-tamper");
std::fs::create_dir_all(&staging_dir).expect("staging dir");
let payload = b"source stream payload".to_vec();
let staging_path = staging_dir.join("0");
std::fs::write(&staging_path, &payload).expect("write staged payload");
let payload_digest = digest_for_bytes("payload.bin", &payload);
let manifest = TransferManifest {
transfer_id: "rqtransfer-trailer-bad".to_string(),
root_name: "payload.bin".to_string(),
is_directory: false,
total_bytes: payload.len() as u64,
merkle_root_hex: sha256_hex_placeholder(),
entries: vec![ManifestEntry {
index: 0,
rel_path: "payload.bin".to_string(),
size: payload.len() as u64,
sha256_hex: sha256_hex_placeholder(),
members: Vec::new(),
fragment: None,
}],
};
let complete = RqRoundComplete {
round_symbols_sent: 0,
entry_digests: vec![ObjectCompleteEntryDigest {
index: 0,
size: payload_digest.size,
sha256_hex: "ff".repeat(32),
}],
logical_digests: vec![ObjectCompleteLogicalDigest {
rel_path: "payload.bin".to_string(),
size: payload_digest.size,
sha256_hex: hex_encode(&payload_digest.content_sha256),
}],
merkle_root_hex: Some(flat_merkle_root_from_digests(&[payload_digest.clone()])),
};
let completion_digests =
CompletionDigestIndex::from_round_complete(&complete, &manifest, true)
.expect("bad digest is still well-formed hex");
let mut decoders = vec![EntryDecoder {
index: 0,
object_id: entry_object_id(&manifest.transfer_id, 0),
size: payload.len() as u64,
pipeline: None,
complete: true,
staging_path,
staging_write_offset: 0,
staging_file_len: payload.len() as u64,
staging_shared: false,
staging_created: true,
staging_file: None,
staging_cursor: None,
staging_unflushed_bytes: 0,
cache_staging_file: false,
bytes_written: payload.len() as u64,
max_block_size: DEFAULT_MAX_BLOCK_SIZE,
source_streaming: true,
source_blocks: Vec::new(),
pending_decodes: Vec::new(),
inc: None,
inc_digest: Some((
payload_digest.size,
payload_digest.content_id,
payload_digest.content_sha256,
)),
source_write_buffer: Vec::new(),
source_write_buffer_offset: None,
}];
let receipt = futures_lite::future::block_on(verify_and_commit(
&manifest,
&mut decoders,
dest.path(),
0,
0,
&BTreeMap::new(),
&completion_digests,
))
.expect("verify returns a fail-closed receipt");
assert!(!receipt.committed, "bad trailer digest must fail closed");
assert!(!receipt.sha_ok);
assert!(!dest.path().join("payload.bin").exists());
}
// ─── E-15 tree coalescing (pack / split) ───────────────────────────────
fn source_entry(dir: &Path, rel: &str, bytes: &[u8]) -> RqSourceEntry {
let abs = dir.join(rel);
if let Some(parent) = abs.parent() {
std::fs::create_dir_all(parent).expect("create parent");
}
std::fs::write(&abs, bytes).expect("write source file");
RqSourceEntry {
rel_path: rel.to_string(),
abs_path: abs,
source_offset: 0,
source_len: None,
members: Vec::new(),
fragment: None,
}
}
#[test]
fn pack_small_files_records_offsets_lens_and_member_sha() {
let dir = tempfile::tempdir().expect("tempdir");
// Three small files (< PACK_THRESHOLD) -> one combined object.
let a = vec![0xAAu8; 100];
let b = vec![0xBBu8; 250];
let c = vec![0xCCu8; 7];
let entries = vec![
source_entry(dir.path(), "d/a", &a),
source_entry(dir.path(), "d/b", &b),
source_entry(dir.path(), "z/c", &c),
];
let config = RqConfig::default();
let (packed, logical_digests, tempdir) =
futures_lite::future::block_on(pack_small_files(entries, &config)).expect("pack");
let _tempdir = tempdir.expect("a pack temp dir was produced");
assert_eq!(
packed.len(),
1,
"three small files coalesce into one object"
);
let pack = &packed[0];
assert_eq!(pack.rel_path, ".atp-pack-0");
assert_eq!(pack.members.len(), 3);
// Members appear in sorted (manifest) order with contiguous offsets.
assert_eq!(pack.members[0].rel_path, "d/a");
assert_eq!(pack.members[0].offset, 0);
assert_eq!(pack.members[0].len, 100);
assert_eq!(pack.members[1].rel_path, "d/b");
assert_eq!(pack.members[1].offset, 100);
assert_eq!(pack.members[1].len, 250);
assert_eq!(pack.members[2].rel_path, "z/c");
assert_eq!(pack.members[2].offset, 350);
assert_eq!(pack.members[2].len, 7);
// Per-member sha matches the file content.
assert_eq!(pack.members[0].sha256_hex, hex_encode(&Sha256::digest(&a)));
assert_eq!(pack.members[1].sha256_hex, hex_encode(&Sha256::digest(&b)));
assert_eq!(pack.members[2].sha256_hex, hex_encode(&Sha256::digest(&c)));
// The temp object is the concatenation in offset order.
let on_disk = std::fs::read(&pack.abs_path).expect("read pack object");
let mut expected = Vec::new();
expected.extend_from_slice(&a);
expected.extend_from_slice(&b);
expected.extend_from_slice(&c);
assert_eq!(on_disk, expected, "pack object is the member concatenation");
// Logical digests cover every logical file (members flattened).
assert_eq!(logical_digests.len(), 3);
let logical_root = flat_merkle_root_from_digests(&logical_digests);
// Same set of {rel_path, content} -> same root as the unpacked files.
let direct: Vec<EntryDigest> = [("d/a", &a), ("d/b", &b), ("z/c", &c)]
.into_iter()
.map(|(rel, bytes)| EntryDigest {
rel_path: rel.to_string(),
size: bytes.len() as u64,
content_id: crate::atp::object::ObjectId::content(
crate::atp::object::ContentId::from_bytes(bytes),
),
content_sha256: Sha256::digest(bytes).into(),
})
.collect();
assert_eq!(
logical_root,
flat_merkle_root_from_digests(&direct),
"logical merkle root is invariant to packing"
);
}
#[test]
fn pack_small_files_coalesces_tree_small_max_size_bucket() {
let dir = tempfile::tempdir().expect("tempdir");
let max_tree_small = vec![0x11u8; 1024 * 1024];
let sibling = vec![0x22u8; 1024 * 1024];
let entries = vec![
source_entry(dir.path(), "leaf/a.bin", &max_tree_small),
source_entry(dir.path(), "leaf/b.bin", &sibling),
];
let config = RqConfig::default();
let (packed, logical_digests, tempdir) =
futures_lite::future::block_on(pack_small_files(entries, &config)).expect("pack");
let _tempdir = tempdir.expect("tree_small max-size entries should pack");
assert_eq!(packed.len(), 1);
let pack = &packed[0];
assert_eq!(pack.members.len(), 2);
assert_eq!(pack.members[0].rel_path, "leaf/a.bin");
assert_eq!(pack.members[0].offset, 0);
assert_eq!(pack.members[0].len, max_tree_small.len() as u64);
assert_eq!(pack.members[1].rel_path, "leaf/b.bin");
assert_eq!(pack.members[1].offset, max_tree_small.len() as u64);
assert_eq!(pack.members[1].len, sibling.len() as u64);
assert_eq!(logical_digests.len(), 2);
assert_eq!(
std::fs::metadata(&pack.abs_path)
.expect("packed object metadata")
.len(),
(max_tree_small.len() + sibling.len()) as u64
);
}
#[test]
fn pack_small_files_leaves_large_files_unpacked_and_root_unchanged() {
let dir = tempfile::tempdir().expect("tempdir");
// Two files >= PACK_THRESHOLD: neither is packed, nothing materialized.
let big1 = vec![1u8; PACK_THRESHOLD as usize];
let big2 = vec![2u8; PACK_THRESHOLD as usize + 13];
let entries = vec![
source_entry(dir.path(), "big1", &big1),
source_entry(dir.path(), "big2", &big2),
];
let config = RqConfig::default();
let (packed, logical_digests, tempdir) =
futures_lite::future::block_on(pack_small_files(entries, &config)).expect("pack");
assert!(tempdir.is_none(), "no packing => no temp dir");
assert_eq!(packed.len(), 2);
assert!(packed.iter().all(|e| e.members.is_empty()));
assert_eq!(packed[0].rel_path, "big1");
assert_eq!(packed[1].rel_path, "big2");
assert_eq!(logical_digests.len(), 2);
// Byte-identical to the per-file digest path the caller would build.
assert_eq!(logical_digests[0].size, big1.len() as u64);
assert_eq!(logical_digests[1].size, big2.len() as u64);
}
#[test]
fn pack_small_files_single_small_file_is_not_packed() {
let dir = tempfile::tempdir().expect("tempdir");
let entries = vec![source_entry(dir.path(), "only", b"tiny")];
let config = RqConfig::default();
let (packed, logical_digests, tempdir) =
futures_lite::future::block_on(pack_small_files(entries, &config)).expect("pack");
assert!(tempdir.is_none(), "a lone small file is not packed");
assert_eq!(packed.len(), 1);
assert!(packed[0].members.is_empty());
assert_eq!(packed[0].rel_path, "only");
assert_eq!(logical_digests.len(), 1);
}
#[test]
fn pack_small_files_respects_configured_object_ceiling() {
let dir = tempfile::tempdir().expect("tempdir");
let too_large_for_one_object = vec![0xA5u8; MAX_SOURCE_BLOCKS + 44];
let tail = vec![0x5Au8; 20];
let entries = vec![
source_entry(dir.path(), "needs-split", &too_large_for_one_object),
source_entry(dir.path(), "tail", &tail),
];
let config = RqConfig {
symbol_size: 1,
max_block_size: 1,
..RqConfig::default()
};
let (packed, logical_digests, tempdir) =
futures_lite::future::block_on(pack_small_files(entries, &config)).expect("pack");
assert!(
tempdir.is_none(),
"packing must not create an unsplittable object above the configured ceiling"
);
assert_eq!(packed.len(), 2);
assert!(packed.iter().all(|entry| entry.members.is_empty()));
let split =
futures_lite::future::block_on(split_large_entries(packed, &logical_digests, &config))
.expect("E-12 split after E-15 pack cap");
assert_eq!(
split
.iter()
.filter(|entry| entry.fragment.is_some())
.count(),
2,
"the over-ceiling small file remains available for ranged object splitting"
);
assert!(split.iter().all(|entry| entry.members.is_empty()));
}
/// End-to-end (in-process) split: build a packed manifest + a staging file
/// holding the member concatenation, then verify_and_commit must split it
/// into the member files on disk, byte-identical.
#[test]
fn verify_and_commit_splits_packed_object_into_members() {
let dest = tempfile::tempdir().expect("dest dir");
let staging_dir = dest.path().join(".atp-rq-test-staging");
std::fs::create_dir_all(&staging_dir).expect("staging dir");
let a = b"first-member-bytes".to_vec();
let b = b"second member, a little longer".to_vec();
let mut object = Vec::new();
object.extend_from_slice(&a);
object.extend_from_slice(&b);
let staging_path = staging_dir.join("0");
std::fs::write(&staging_path, &object).expect("write packed staging object");
let members = vec![
PackedMember {
rel_path: "dir/a.txt".to_string(),
offset: 0,
len: a.len() as u64,
sha256_hex: hex_encode(&Sha256::digest(&a)),
},
PackedMember {
rel_path: "dir/sub/b.txt".to_string(),
offset: a.len() as u64,
len: b.len() as u64,
sha256_hex: hex_encode(&Sha256::digest(&b)),
},
];
// Merkle root over the LOGICAL files (what the sender computes).
let logical: Vec<EntryDigest> = [("dir/a.txt", &a), ("dir/sub/b.txt", &b)]
.into_iter()
.map(|(rel, bytes)| EntryDigest {
rel_path: rel.to_string(),
size: bytes.len() as u64,
content_id: crate::atp::object::ObjectId::content(
crate::atp::object::ContentId::from_bytes(bytes),
),
content_sha256: Sha256::digest(bytes).into(),
})
.collect();
let merkle_root_hex = flat_merkle_root_from_digests(&logical);
let object_sha = hex_encode(&Sha256::digest(&object));
let manifest = TransferManifest {
transfer_id: "rqtransfer1".to_string(),
root_name: "payload".to_string(),
is_directory: true,
total_bytes: object.len() as u64,
merkle_root_hex,
entries: vec![ManifestEntry {
index: 0,
rel_path: ".atp-pack-0".to_string(),
size: object.len() as u64,
sha256_hex: object_sha,
members,
fragment: None,
}],
};
let mut decoders = vec![EntryDecoder {
index: 0,
object_id: entry_object_id(&manifest.transfer_id, 0),
size: object.len() as u64,
pipeline: None,
complete: true,
staging_path,
staging_write_offset: 0,
staging_file_len: object.len() as u64,
staging_shared: false,
staging_created: true,
staging_file: None,
staging_cursor: None,
staging_unflushed_bytes: 0,
cache_staging_file: false,
bytes_written: object.len() as u64,
max_block_size: DEFAULT_MAX_BLOCK_SIZE,
source_streaming: false,
source_blocks: Vec::new(),
pending_decodes: Vec::new(),
inc: None,
inc_digest: None,
source_write_buffer: Vec::new(),
source_write_buffer_offset: None,
}];
let receipt = futures_lite::future::block_on(verify_and_commit(
&manifest,
&mut decoders,
dest.path(),
0,
0,
&std::collections::BTreeMap::new(),
&CompletionDigestIndex::default(),
))
.expect("verify_and_commit");
assert!(
receipt.committed,
"packed transfer must commit: {receipt:?}"
);
assert!(receipt.sha_ok);
assert!(receipt.merkle_ok);
assert_eq!(receipt.files, 2, "two LOGICAL files delivered");
let out_a = dest.path().join("payload/dir/a.txt");
let out_b = dest.path().join("payload/dir/sub/b.txt");
assert_eq!(std::fs::read(&out_a).expect("member a"), a);
assert_eq!(std::fs::read(&out_b).expect("member b"), b);
// The synthetic packed object name must not appear on disk.
assert!(!dest.path().join("payload/.atp-pack-0").exists());
}
#[test]
fn packed_member_streaming_helpers_reuse_small_buffers() {
let temp = tempfile::tempdir().expect("tempdir");
let staging_path = temp.path().join("pack-object");
let a = b"alpha-member".to_vec();
let b = b"beta-member-is-longer".to_vec();
let mut packed = Vec::new();
packed.extend_from_slice(&a);
packed.extend_from_slice(&b);
std::fs::write(&staging_path, &packed).expect("write packed staging object");
let members = vec![
PackedMember {
rel_path: "a.txt".to_string(),
offset: 0,
len: a.len() as u64,
sha256_hex: hex_encode(&Sha256::digest(&a)),
},
PackedMember {
rel_path: "nested/b.txt".to_string(),
offset: a.len() as u64,
len: b.len() as u64,
sha256_hex: hex_encode(&Sha256::digest(&b)),
},
];
let mut digests = Vec::new();
let mut logical_files = 0;
let mut verify_buf = [0u8; 3];
let ok = futures_lite::future::block_on(hash_packed_members_streaming(
&staging_path,
&members,
&mut digests,
&mut logical_files,
&mut verify_buf,
))
.expect("streaming member verification");
assert!(ok);
assert_eq!(logical_files, 2);
assert_eq!(digests.len(), 2);
assert_eq!(digests[0].rel_path, "a.txt");
let expected_a_sha: [u8; 32] = Sha256::digest(&a).into();
assert_eq!(digests[0].content_sha256, expected_a_sha);
assert_eq!(digests[1].rel_path, "nested/b.txt");
let expected_b_sha: [u8; 32] = Sha256::digest(&b).into();
assert_eq!(digests[1].content_sha256, expected_b_sha);
let out_root = temp.path().join("out");
let writes = vec![
PackedMemberWrite {
offset: members[0].offset,
len: members[0].len,
out_path: out_root.join(&members[0].rel_path),
},
PackedMemberWrite {
offset: members[1].offset,
len: members[1].len,
out_path: out_root.join(&members[1].rel_path),
},
];
let mut write_buf = [0u8; 4];
futures_lite::future::block_on(write_packed_member_batch(
&staging_path,
&writes,
&mut write_buf,
))
.expect("batched member commit");
assert_eq!(std::fs::read(out_root.join("a.txt")).expect("read a"), a);
assert_eq!(
std::fs::read(out_root.join("nested/b.txt")).expect("read b"),
b
);
}
/// MATRIX-211 regression: the one-shot packed commit must slice members
/// relative to the batch SPAN (min offset), not the staging file start,
/// and must not care about member order; a single member takes the
/// streaming fallback and still commits byte-identically.
#[test]
fn packed_member_batch_oneshot_spans_and_fallback_commit_identically() {
let temp = tempfile::tempdir().expect("tempdir");
let staging_path = temp.path().join("pack-object");
let pad = vec![0xEE_u8; 7];
let a = b"span-member-a".to_vec();
let b = b"span-member-b-longer".to_vec();
let mut packed = Vec::new();
packed.extend_from_slice(&pad);
packed.extend_from_slice(&a);
packed.extend_from_slice(&b);
std::fs::write(&staging_path, &packed).expect("write packed staging object");
let a_off = pad.len() as u64;
let b_off = a_off + a.len() as u64;
// Out-of-order members with span_start > 0 → one-shot path.
let out_root = temp.path().join("out");
let writes = vec![
PackedMemberWrite {
offset: b_off,
len: b.len() as u64,
out_path: out_root.join("deep/nested/b.bin"),
},
PackedMemberWrite {
offset: a_off,
len: a.len() as u64,
out_path: out_root.join("a.bin"),
},
];
let mut write_buf = [0u8; 4];
futures_lite::future::block_on(write_packed_member_batch(
&staging_path,
&writes,
&mut write_buf,
))
.expect("one-shot span commit");
assert_eq!(std::fs::read(out_root.join("a.bin")).expect("read a"), a);
assert_eq!(
std::fs::read(out_root.join("deep/nested/b.bin")).expect("read b"),
b
);
// Single member → streaming fallback path.
let solo = vec![PackedMemberWrite {
offset: a_off,
len: a.len() as u64,
out_path: out_root.join("solo/a-again.bin"),
}];
futures_lite::future::block_on(write_packed_member_batch(
&staging_path,
&solo,
&mut write_buf,
))
.expect("single-member fallback commit");
assert_eq!(
std::fs::read(out_root.join("solo/a-again.bin")).expect("read solo"),
a
);
}
#[test]
fn validate_manifest_checks_packed_member_table() {
// A well-formed packed manifest entry is accepted.
let good_members = vec![
PackedMember {
rel_path: "dir/a".to_string(),
offset: 0,
len: 10,
sha256_hex: "aa".repeat(32),
},
PackedMember {
rel_path: "dir/b".to_string(),
offset: 10,
len: 5,
sha256_hex: "bb".repeat(32),
},
];
let ok_entry = ManifestEntry {
index: 0,
rel_path: ".atp-pack-0".to_string(),
size: 15,
sha256_hex: "cc".repeat(32),
members: good_members.clone(),
fragment: None,
};
assert!(
validate_manifest(&manifest_with(vec![ok_entry], 15), &RqConfig::default()).is_ok()
);
// Non-contiguous offsets fail closed.
let mut gap = good_members.clone();
gap[1].offset = 11;
let entry = ManifestEntry {
index: 0,
rel_path: ".atp-pack-0".to_string(),
size: 15,
sha256_hex: "cc".repeat(32),
members: gap,
fragment: None,
};
assert!(matches!(
validate_manifest(&manifest_with(vec![entry], 15), &RqConfig::default()),
Err(RqError::Frame(m)) if m.contains("not contiguous")
));
// Member lengths must tile the object exactly.
let entry = ManifestEntry {
index: 0,
rel_path: ".atp-pack-0".to_string(),
size: 99,
sha256_hex: "cc".repeat(32),
members: good_members.clone(),
fragment: None,
};
assert!(matches!(
validate_manifest(&manifest_with(vec![entry], 99), &RqConfig::default()),
Err(RqError::Frame(m)) if m.contains("members cover")
));
// An unsafe member rel_path fails closed.
let mut evil = good_members;
evil[1].rel_path = "../escape".to_string();
let entry = ManifestEntry {
index: 0,
rel_path: ".atp-pack-0".to_string(),
size: 15,
sha256_hex: "cc".repeat(32),
members: evil,
fragment: None,
};
assert!(matches!(
validate_manifest(&manifest_with(vec![entry], 15), &RqConfig::default()),
Err(RqError::Source(m)) if m.contains("unsafe manifest rel_path")
));
}
/// A corrupted member sha must fail closed: nothing is committed/written.
#[test]
fn verify_and_commit_rejects_packed_object_with_wrong_member_sha() {
let dest = tempfile::tempdir().expect("dest dir");
let staging_dir = dest.path().join(".atp-rq-test-staging");
std::fs::create_dir_all(&staging_dir).expect("staging dir");
let a = b"member-one".to_vec();
let b = b"member-two".to_vec();
let mut object = Vec::new();
object.extend_from_slice(&a);
object.extend_from_slice(&b);
let staging_path = staging_dir.join("0");
std::fs::write(&staging_path, &object).expect("write packed staging object");
// Build a correct logical merkle root, but lie about member b's sha so
// the per-member check fails (object sha + merkle stay self-consistent).
let logical: Vec<EntryDigest> = [("a.txt", &a), ("b.txt", &b)]
.into_iter()
.map(|(rel, bytes)| EntryDigest {
rel_path: rel.to_string(),
size: bytes.len() as u64,
content_id: crate::atp::object::ObjectId::content(
crate::atp::object::ContentId::from_bytes(bytes),
),
content_sha256: Sha256::digest(bytes).into(),
})
.collect();
let merkle_root_hex = flat_merkle_root_from_digests(&logical);
let manifest = TransferManifest {
transfer_id: "rqtransfer1".to_string(),
root_name: "payload".to_string(),
is_directory: true,
total_bytes: object.len() as u64,
merkle_root_hex,
entries: vec![ManifestEntry {
index: 0,
rel_path: ".atp-pack-0".to_string(),
size: object.len() as u64,
sha256_hex: hex_encode(&Sha256::digest(&object)),
members: vec![
PackedMember {
rel_path: "a.txt".to_string(),
offset: 0,
len: a.len() as u64,
sha256_hex: hex_encode(&Sha256::digest(&a)),
},
PackedMember {
rel_path: "b.txt".to_string(),
offset: a.len() as u64,
len: b.len() as u64,
// WRONG sha for member b.
sha256_hex: "ff".repeat(32),
},
],
fragment: None,
}],
};
let mut decoders = vec![EntryDecoder {
index: 0,
object_id: entry_object_id(&manifest.transfer_id, 0),
size: object.len() as u64,
pipeline: None,
complete: true,
staging_path,
staging_write_offset: 0,
staging_file_len: object.len() as u64,
staging_shared: false,
staging_created: true,
staging_file: None,
staging_cursor: None,
staging_unflushed_bytes: 0,
cache_staging_file: false,
bytes_written: object.len() as u64,
max_block_size: DEFAULT_MAX_BLOCK_SIZE,
source_streaming: false,
source_blocks: Vec::new(),
pending_decodes: Vec::new(),
inc: None,
inc_digest: None,
source_write_buffer: Vec::new(),
source_write_buffer_offset: None,
}];
let receipt = futures_lite::future::block_on(verify_and_commit(
&manifest,
&mut decoders,
dest.path(),
0,
0,
&std::collections::BTreeMap::new(),
&CompletionDigestIndex::default(),
))
.expect("verify_and_commit returns a receipt");
assert!(!receipt.committed, "wrong member sha must fail closed");
assert!(!receipt.sha_ok);
// Nothing written into place.
assert!(!dest.path().join("payload/a.txt").exists());
assert!(!dest.path().join("payload/b.txt").exists());
}
}