use bytes::{Bytes, BytesMut};
use std::io::IoSlice;
use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
use std::time::{Duration, Instant};
pub const DEFAULT_COALESCE_CAPACITY: usize = 4096;
pub const DEFAULT_FLUSH_THRESHOLD: usize = 16384;
pub const MIN_COALESCE_SIZE: usize = 512;
pub const MAX_COALESCE_BUFFER: usize = 1024 * 1024;
pub const DEFAULT_FLUSH_TIMEOUT_US: u64 = 100;
#[derive(Debug, Clone)]
pub struct CoalesceConfig {
pub initial_capacity: usize,
pub flush_threshold: usize,
pub max_buffer_size: usize,
pub bypass_threshold: usize,
pub flush_timeout_us: u64,
pub use_tcp_cork: bool,
pub collect_stats: bool,
}
impl Default for CoalesceConfig {
fn default() -> Self {
Self {
initial_capacity: DEFAULT_COALESCE_CAPACITY,
flush_threshold: DEFAULT_FLUSH_THRESHOLD,
max_buffer_size: MAX_COALESCE_BUFFER,
bypass_threshold: 65536, flush_timeout_us: DEFAULT_FLUSH_TIMEOUT_US,
use_tcp_cork: true,
collect_stats: true,
}
}
}
impl CoalesceConfig {
pub fn high_throughput() -> Self {
Self {
initial_capacity: 8192,
flush_threshold: 32768,
max_buffer_size: MAX_COALESCE_BUFFER,
bypass_threshold: 131072, flush_timeout_us: 500, use_tcp_cork: true,
collect_stats: false,
}
}
pub fn low_latency() -> Self {
Self {
initial_capacity: 2048,
flush_threshold: 4096,
max_buffer_size: 65536,
bypass_threshold: 16384, flush_timeout_us: 10, use_tcp_cork: false,
collect_stats: false,
}
}
pub fn memory_efficient() -> Self {
Self {
initial_capacity: 1024,
flush_threshold: 8192,
max_buffer_size: 65536,
bypass_threshold: 32768,
flush_timeout_us: 200,
use_tcp_cork: true,
collect_stats: true,
}
}
}
#[derive(Debug)]
pub struct WriteCoalescer {
buffer: BytesMut,
config: CoalesceConfig,
writes_coalesced: usize,
first_write_time: Option<Instant>,
total_bytes: usize,
}
impl WriteCoalescer {
pub fn new(config: CoalesceConfig) -> Self {
Self {
buffer: BytesMut::with_capacity(config.initial_capacity),
config,
writes_coalesced: 0,
first_write_time: None,
total_bytes: 0,
}
}
pub fn default_config() -> Self {
Self::new(CoalesceConfig::default())
}
#[inline]
pub fn write(&mut self, data: &[u8]) -> WriteResult {
if data.is_empty() {
return WriteResult::Buffered;
}
if data.len() >= self.config.bypass_threshold {
COALESCE_STATS.record_bypass(data.len());
return WriteResult::Bypass(Bytes::copy_from_slice(data));
}
if self.first_write_time.is_none() {
self.first_write_time = Some(Instant::now());
}
self.buffer.extend_from_slice(data);
self.writes_coalesced += 1;
self.total_bytes += data.len();
if self.config.collect_stats {
COALESCE_STATS.record_coalesce(data.len());
}
if self.should_flush() {
WriteResult::ShouldFlush
} else {
WriteResult::Buffered
}
}
#[inline]
pub fn write_bytes(&mut self, data: Bytes) -> WriteResult {
if data.is_empty() {
return WriteResult::Buffered;
}
if data.len() >= self.config.bypass_threshold {
COALESCE_STATS.record_bypass(data.len());
return WriteResult::Bypass(data);
}
if self.first_write_time.is_none() {
self.first_write_time = Some(Instant::now());
}
self.buffer.extend_from_slice(&data);
self.writes_coalesced += 1;
self.total_bytes += data.len();
if self.config.collect_stats {
COALESCE_STATS.record_coalesce(data.len());
}
if self.should_flush() {
WriteResult::ShouldFlush
} else {
WriteResult::Buffered
}
}
#[inline]
pub fn should_flush(&self) -> bool {
if self.buffer.len() >= self.config.flush_threshold {
return true;
}
if self.buffer.len() >= self.config.max_buffer_size {
return true;
}
if self.config.flush_timeout_us > 0
&& let Some(first_time) = self.first_write_time
{
let elapsed_us = first_time.elapsed().as_micros() as u64;
if elapsed_us >= self.config.flush_timeout_us {
return true;
}
}
false
}
#[inline]
pub fn must_flush(&self) -> bool {
self.buffer.len() >= self.config.max_buffer_size
}
#[inline]
pub fn len(&self) -> usize {
self.buffer.len()
}
#[inline]
pub fn is_empty(&self) -> bool {
self.buffer.is_empty()
}
#[inline]
pub fn writes_coalesced(&self) -> usize {
self.writes_coalesced
}
#[inline]
pub fn total_bytes(&self) -> usize {
self.total_bytes
}
#[inline]
pub fn remaining_capacity(&self) -> usize {
self.config
.flush_threshold
.saturating_sub(self.buffer.len())
}
#[inline]
pub fn take(&mut self) -> Bytes {
let writes = self.writes_coalesced;
let bytes = self.buffer.len();
let data = self.buffer.split().freeze();
self.writes_coalesced = 0;
self.first_write_time = None;
if self.config.collect_stats && bytes > 0 {
COALESCE_STATS.record_flush(writes, bytes);
}
data
}
#[inline]
pub fn take_mut(&mut self) -> BytesMut {
let writes = self.writes_coalesced;
let bytes = self.buffer.len();
let data = self.buffer.split();
self.writes_coalesced = 0;
self.first_write_time = None;
if self.config.collect_stats && bytes > 0 {
COALESCE_STATS.record_flush(writes, bytes);
}
data
}
#[inline]
pub fn peek(&self) -> &[u8] {
&self.buffer
}
#[inline]
pub fn clear(&mut self) {
self.buffer.clear();
self.writes_coalesced = 0;
self.first_write_time = None;
}
pub fn reset(&mut self) {
self.buffer.clear();
self.writes_coalesced = 0;
self.first_write_time = None;
self.total_bytes = 0;
}
#[inline]
pub fn config(&self) -> &CoalesceConfig {
&self.config
}
#[inline]
pub fn reserve(&mut self, additional: usize) {
self.buffer.reserve(additional);
}
pub fn time_since_first_write(&self) -> Option<Duration> {
self.first_write_time.map(|t| t.elapsed())
}
}
#[derive(Debug)]
pub enum WriteResult {
Buffered,
ShouldFlush,
Bypass(Bytes),
}
impl WriteResult {
#[inline]
pub fn is_buffered(&self) -> bool {
matches!(self, Self::Buffered)
}
#[inline]
pub fn should_flush(&self) -> bool {
matches!(self, Self::ShouldFlush)
}
#[inline]
pub fn is_bypass(&self) -> bool {
matches!(self, Self::Bypass(_))
}
#[inline]
pub fn take_bypass(self) -> Option<Bytes> {
match self {
Self::Bypass(data) => Some(data),
_ => None,
}
}
}
#[derive(Debug)]
pub struct MultiBufferCoalescer {
headers: WriteCoalescer,
body: WriteCoalescer,
trailers: WriteCoalescer,
}
impl MultiBufferCoalescer {
pub fn new(config: CoalesceConfig) -> Self {
Self {
headers: WriteCoalescer::new(CoalesceConfig {
initial_capacity: 1024,
flush_threshold: 4096,
max_buffer_size: 16384,
bypass_threshold: 8192,
..config.clone()
}),
body: WriteCoalescer::new(config.clone()),
trailers: WriteCoalescer::new(CoalesceConfig {
initial_capacity: 256,
flush_threshold: 1024,
max_buffer_size: 4096,
bypass_threshold: 2048,
..config
}),
}
}
#[inline]
pub fn write_header(&mut self, data: &[u8]) -> WriteResult {
self.headers.write(data)
}
#[inline]
pub fn write_header_line(&mut self, name: &str, value: &str) {
self.headers.buffer.extend_from_slice(name.as_bytes());
self.headers.buffer.extend_from_slice(b": ");
self.headers.buffer.extend_from_slice(value.as_bytes());
self.headers.buffer.extend_from_slice(b"\r\n");
self.headers.writes_coalesced += 1;
}
#[inline]
pub fn write_body(&mut self, data: &[u8]) -> WriteResult {
self.body.write(data)
}
#[inline]
pub fn write_trailer(&mut self, data: &[u8]) -> WriteResult {
self.trailers.write(data)
}
#[inline]
pub fn should_flush(&self) -> bool {
self.headers.should_flush() || self.body.should_flush() || self.trailers.should_flush()
}
#[inline]
pub fn total_len(&self) -> usize {
self.headers.len() + self.body.len() + self.trailers.len()
}
pub fn take_combined(&mut self) -> Bytes {
let total = self.total_len();
if total == 0 {
return Bytes::new();
}
let mut combined = BytesMut::with_capacity(total);
combined.extend_from_slice(self.headers.peek());
combined.extend_from_slice(self.body.peek());
combined.extend_from_slice(self.trailers.peek());
self.headers.clear();
self.body.clear();
self.trailers.clear();
combined.freeze()
}
pub fn as_io_slices(&self) -> Vec<IoSlice<'_>> {
let mut slices = Vec::with_capacity(3);
if !self.headers.is_empty() {
slices.push(IoSlice::new(self.headers.peek()));
}
if !self.body.is_empty() {
slices.push(IoSlice::new(self.body.peek()));
}
if !self.trailers.is_empty() {
slices.push(IoSlice::new(self.trailers.peek()));
}
slices
}
pub fn reset(&mut self) {
self.headers.reset();
self.body.reset();
self.trailers.reset();
}
}
#[derive(Debug)]
pub struct ConnectionWriteBuffer {
coalescer: WriteCoalescer,
pending_large: Vec<Bytes>,
#[allow(dead_code)]
connection_id: u64,
flushes: usize,
}
impl ConnectionWriteBuffer {
pub fn new(connection_id: u64, config: CoalesceConfig) -> Self {
Self {
coalescer: WriteCoalescer::new(config),
pending_large: Vec::new(),
connection_id,
flushes: 0,
}
}
#[inline]
pub fn write(&mut self, data: &[u8]) {
if let WriteResult::Bypass(bytes) = self.coalescer.write(data) {
self.push_large(bytes);
}
}
#[inline]
pub fn write_bytes(&mut self, data: Bytes) {
if let WriteResult::Bypass(bytes) = self.coalescer.write_bytes(data) {
self.push_large(bytes);
}
}
#[inline]
fn push_large(&mut self, bytes: Bytes) {
if !self.coalescer.is_empty() {
let coalesced = self.coalescer.take();
self.pending_large.push(coalesced);
}
self.pending_large.push(bytes);
}
#[inline]
pub fn should_flush(&self) -> bool {
!self.pending_large.is_empty() || self.coalescer.should_flush()
}
pub fn take_all(&mut self) -> Vec<Bytes> {
let mut result = std::mem::take(&mut self.pending_large);
if !self.coalescer.is_empty() {
result.push(self.coalescer.take());
}
self.flushes += 1;
result
}
pub fn as_io_slices(&self) -> Vec<IoSlice<'_>> {
let mut slices = Vec::with_capacity(1 + self.pending_large.len());
for large in &self.pending_large {
slices.push(IoSlice::new(large));
}
if !self.coalescer.is_empty() {
slices.push(IoSlice::new(self.coalescer.peek()));
}
slices
}
#[inline]
pub fn pending_bytes(&self) -> usize {
let large_bytes: usize = self.pending_large.iter().map(|b| b.len()).sum();
self.coalescer.len() + large_bytes
}
#[inline]
pub fn flushes(&self) -> usize {
self.flushes
}
pub fn reset(&mut self) {
self.coalescer.reset();
self.pending_large.clear();
}
}
#[derive(Debug, Default)]
pub struct CoalesceStats {
coalesced: AtomicU64,
bytes_coalesced: AtomicU64,
bypassed: AtomicU64,
bytes_bypassed: AtomicU64,
flushes: AtomicU64,
writes_per_flush_sum: AtomicU64,
max_writes_per_flush: AtomicUsize,
}
impl CoalesceStats {
pub fn new() -> Self {
Self::default()
}
#[inline]
pub fn record_coalesce(&self, bytes: usize) {
self.coalesced.fetch_add(1, Ordering::Relaxed);
self.bytes_coalesced
.fetch_add(bytes as u64, Ordering::Relaxed);
}
#[inline]
pub fn record_bypass(&self, bytes: usize) {
self.bypassed.fetch_add(1, Ordering::Relaxed);
self.bytes_bypassed
.fetch_add(bytes as u64, Ordering::Relaxed);
}
#[inline]
pub fn record_flush(&self, writes: usize, _bytes: usize) {
self.flushes.fetch_add(1, Ordering::Relaxed);
self.writes_per_flush_sum
.fetch_add(writes as u64, Ordering::Relaxed);
self.max_writes_per_flush
.fetch_max(writes, Ordering::Relaxed);
}
pub fn coalesced(&self) -> u64 {
self.coalesced.load(Ordering::Relaxed)
}
pub fn bytes_coalesced(&self) -> u64 {
self.bytes_coalesced.load(Ordering::Relaxed)
}
pub fn bypassed(&self) -> u64 {
self.bypassed.load(Ordering::Relaxed)
}
pub fn bytes_bypassed(&self) -> u64 {
self.bytes_bypassed.load(Ordering::Relaxed)
}
pub fn flushes(&self) -> u64 {
self.flushes.load(Ordering::Relaxed)
}
pub fn avg_writes_per_flush(&self) -> f64 {
let flushes = self.flushes();
let sum = self.writes_per_flush_sum.load(Ordering::Relaxed);
if flushes > 0 {
sum as f64 / flushes as f64
} else {
0.0
}
}
pub fn max_writes_per_flush(&self) -> usize {
self.max_writes_per_flush.load(Ordering::Relaxed)
}
pub fn coalesce_ratio(&self) -> f64 {
let coalesced = self.coalesced();
let bypassed = self.bypassed();
let total = coalesced + bypassed;
if total > 0 {
(coalesced as f64 / total as f64) * 100.0
} else {
0.0
}
}
pub fn syscall_reduction_ratio(&self) -> f64 {
let writes = self.coalesced();
let flushes = self.flushes();
if writes > 0 {
let saved = writes.saturating_sub(flushes);
(saved as f64 / writes as f64) * 100.0
} else {
0.0
}
}
}
static COALESCE_STATS: CoalesceStats = CoalesceStats {
coalesced: AtomicU64::new(0),
bytes_coalesced: AtomicU64::new(0),
bypassed: AtomicU64::new(0),
bytes_bypassed: AtomicU64::new(0),
flushes: AtomicU64::new(0),
writes_per_flush_sum: AtomicU64::new(0),
max_writes_per_flush: AtomicUsize::new(0),
};
pub fn coalesce_stats() -> &'static CoalesceStats {
&COALESCE_STATS
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_coalesce_config_default() {
let config = CoalesceConfig::default();
assert_eq!(config.initial_capacity, DEFAULT_COALESCE_CAPACITY);
assert_eq!(config.flush_threshold, DEFAULT_FLUSH_THRESHOLD);
}
#[test]
fn test_coalesce_config_presets() {
let high = CoalesceConfig::high_throughput();
assert!(high.flush_threshold > DEFAULT_FLUSH_THRESHOLD);
let low = CoalesceConfig::low_latency();
assert!(low.flush_threshold < DEFAULT_FLUSH_THRESHOLD);
}
#[test]
fn test_write_coalescer_basic() {
let mut coalescer = WriteCoalescer::new(CoalesceConfig::default());
let result = coalescer.write(b"Hello");
assert!(!result.is_bypass());
assert_eq!(coalescer.len(), 5);
coalescer.write(b", World!");
assert_eq!(coalescer.len(), 13);
assert_eq!(coalescer.writes_coalesced(), 2);
let data = coalescer.take();
assert_eq!(&data[..], b"Hello, World!");
assert!(coalescer.is_empty());
}
#[test]
fn test_write_coalescer_bypass() {
let config = CoalesceConfig {
bypass_threshold: 10,
..Default::default()
};
let mut coalescer = WriteCoalescer::new(config);
let result = coalescer.write(b"This is a large write that exceeds threshold");
assert!(result.is_bypass());
assert!(coalescer.is_empty()); }
#[test]
fn test_write_coalescer_flush_threshold() {
let config = CoalesceConfig {
flush_threshold: 20,
..Default::default()
};
let mut coalescer = WriteCoalescer::new(config);
coalescer.write(b"12345");
assert!(!coalescer.should_flush());
coalescer.write(b"1234567890");
assert!(!coalescer.should_flush());
coalescer.write(b"12345");
assert!(coalescer.should_flush()); }
#[test]
fn test_write_result_methods() {
let buffered = WriteResult::Buffered;
assert!(buffered.is_buffered());
assert!(!buffered.should_flush());
assert!(!buffered.is_bypass());
let should_flush = WriteResult::ShouldFlush;
assert!(!should_flush.is_buffered());
assert!(should_flush.should_flush());
let bypass = WriteResult::Bypass(Bytes::from_static(b"test"));
assert!(bypass.is_bypass());
if let Some(data) = bypass.take_bypass() {
assert_eq!(&data[..], b"test");
}
}
#[test]
fn test_multi_buffer_coalescer() {
let mut coalescer = MultiBufferCoalescer::new(CoalesceConfig::default());
coalescer.write_header(b"HTTP/1.1 200 OK\r\n");
coalescer.write_header_line("Content-Type", "text/plain");
coalescer.write_header(b"\r\n");
coalescer.write_body(b"Hello, World!");
assert!(coalescer.total_len() > 0);
let slices = coalescer.as_io_slices();
assert_eq!(slices.len(), 2);
let combined = coalescer.take_combined();
assert!(!combined.is_empty());
}
#[test]
fn test_connection_write_buffer() {
let mut buffer = ConnectionWriteBuffer::new(1, CoalesceConfig::default());
buffer.write(b"Small write 1");
buffer.write(b"Small write 2");
buffer.write(b"Small write 3");
assert!(buffer.pending_bytes() > 0);
let data = buffer.take_all();
assert!(!data.is_empty());
assert_eq!(buffer.flushes(), 1);
}
#[test]
fn test_connection_write_buffer_large_bypass() {
let config = CoalesceConfig {
bypass_threshold: 10,
..Default::default()
};
let mut buffer = ConnectionWriteBuffer::new(1, config);
buffer.write(b"Small");
buffer.write(b"This is a large write that will be bypassed");
let slices = buffer.as_io_slices();
assert_eq!(slices.len(), 2);
}
#[test]
fn test_connection_write_buffer_interleaved_ordering() {
let config = CoalesceConfig {
bypass_threshold: 64,
..Default::default()
};
let mut buffer = ConnectionWriteBuffer::new(1, config);
let large = vec![b'X'; 128];
buffer.write(b"header");
buffer.write(&large);
buffer.write(b"trailer");
let flattened: Vec<u8> = buffer
.as_io_slices()
.iter()
.flat_map(|s| s.iter().copied())
.collect();
let mut expected = b"header".to_vec();
expected.extend_from_slice(&large);
expected.extend_from_slice(b"trailer");
assert_eq!(flattened, expected);
let taken: Vec<u8> = buffer
.take_all()
.iter()
.flat_map(|b| b.iter().copied())
.collect();
assert_eq!(taken, expected);
}
#[test]
fn test_coalesce_stats() {
let stats = coalesce_stats();
let _ = stats.coalesced();
let _ = stats.bypassed();
let _ = stats.flushes();
let _ = stats.coalesce_ratio();
let _ = stats.syscall_reduction_ratio();
}
#[test]
fn test_take_mut() {
let mut coalescer = WriteCoalescer::new(CoalesceConfig::default());
coalescer.write(b"Hello");
let mut buf = coalescer.take_mut();
buf.extend_from_slice(b", World!");
assert_eq!(&buf[..], b"Hello, World!");
assert!(coalescer.is_empty());
}
#[test]
fn test_reserve() {
let mut coalescer = WriteCoalescer::new(CoalesceConfig::default());
coalescer.reserve(10000);
coalescer.write(b"Now we have plenty of space");
assert!(coalescer.len() < 10000);
}
}