use crate::inputs::Split;
use crate::to_u64;
use std::sync::{Arc, Mutex};
#[derive(Clone, Copy, Debug)]
pub(crate) struct BufRange {
pub pos: u64,
pub len: u16,
}
pub(crate) struct RingBuf {
buf: Box<[u8]>,
start: u64,
end: u64,
}
#[allow(dead_code)] impl RingBuf {
pub fn new(capacity: usize) -> Self {
let capacity = capacity.max(1).checked_next_power_of_two().unwrap();
Self {
buf: vec![0; capacity].into_boxed_slice(),
start: 0,
end: 0,
}
}
#[inline]
fn mask(&self) -> usize {
self.buf.len() - 1
}
#[inline]
pub fn start(&self) -> u64 {
self.start
}
#[inline]
pub fn end(&self) -> u64 {
self.end
}
#[inline]
pub fn len(&self) -> usize {
(self.end - self.start) as usize
}
#[inline]
pub fn is_empty(&self) -> bool {
self.start == self.end
}
#[inline]
pub fn capacity(&self) -> usize {
self.buf.len()
}
#[inline]
pub fn available(&self) -> usize {
self.buf.len() - self.len()
}
pub fn split(&self, pos: u64, len: usize) -> Split<'_> {
assert!(
pos >= self.start && pos + to_u64(len) <= self.end,
"split({pos}, {len}) outside valid range [{}, {})",
self.start,
self.end,
);
let i = pos as usize & self.mask();
let contiguous = self.buf.len() - i;
if let Some(second_len) = len.checked_sub(contiguous) {
Split::new(&self.buf[i..], &self.buf[..second_len])
} else {
Split::new(&self.buf[i..i + len], &[])
}
}
pub fn data_split(&self) -> Split<'_> {
if self.is_empty() {
return Split::new(&[], &[]);
}
self.split(self.start, self.len())
}
pub fn advance_to(&mut self, pos: u64) {
assert!(
self.start <= pos && pos <= self.end,
"advance_to({pos}): must be within [{}, {}]",
self.start,
self.end,
);
self.start = pos;
}
pub fn spare_capacity(&mut self, reserve: usize) -> (&mut [u8], &mut [u8]) {
self.reserve(reserve);
if self.len() == self.buf.len() {
return (&mut [], &mut []);
}
let mask = self.mask();
let end_i = self.end as usize & mask;
let start_i = self.start as usize & mask;
if end_i < start_i {
(&mut self.buf[end_i..start_i], &mut [])
} else {
let (head, tail) = self.buf.split_at_mut(end_i);
(tail, &mut head[..start_i])
}
}
pub fn advance_end(&mut self, n: usize) {
assert!(
n <= self.available(),
"advance_end({n}): only {} available",
self.available(),
);
self.end += to_u64(n);
}
#[cfg(test)]
pub fn extend(&mut self, data: &[u8]) {
let (s1, s2) = self.spare_capacity(data.len());
let mid = data.len().min(s1.len());
s1[..mid].copy_from_slice(&data[..mid]);
s2[..data.len() - mid].copy_from_slice(&data[mid..]);
self.advance_end(data.len());
}
#[inline]
fn reserve(&mut self, additional: usize) {
if additional > self.available() {
self.realloc(additional);
}
}
#[cold]
fn realloc(&mut self, additional: usize) {
let data_len = self.len();
let new_size = data_len
.checked_add(additional)
.and_then(usize::checked_next_power_of_two)
.unwrap();
debug_assert!(new_size > self.buf.len());
let mut new_buf = vec![0u8; new_size].into_boxed_slice();
if data_len > 0 {
let (s1, s2) = self.split(self.start, data_len).slices();
let mut i = self.start as usize & (new_size - 1);
for src in [s1, s2] {
let contiguous = new_size - i;
if let Some(second_len) = src.len().checked_sub(contiguous) {
new_buf[i..].copy_from_slice(&src[..contiguous]);
new_buf[..second_len].copy_from_slice(&src[contiguous..]);
i = second_len;
} else {
new_buf[i..i + src.len()].copy_from_slice(src);
i += src.len();
}
}
}
self.buf = new_buf;
}
}
impl std::fmt::Debug for RingBuf {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("RingBuf")
.field("start", &self.start)
.field("end", &self.end)
.field("capacity", &self.buf.len())
.finish()
}
}
#[derive(Debug)]
struct MarkRegistry {
marks: Mutex<Vec<u64>>,
}
impl MarkRegistry {
fn new() -> Arc<Self> {
Arc::new(Self {
marks: Mutex::new(Vec::new()),
})
}
fn add(&self, pos: u64) {
let mut marks = self.marks.lock().unwrap();
let i = marks.partition_point(|&m| m < pos);
marks.insert(i, pos);
}
fn remove(&self, pos: u64) {
let mut marks = self.marks.lock().unwrap();
if let Ok(i) = marks.binary_search(&pos) {
marks.remove(i);
} else {
panic!("Mark::drop: no mark at position {pos}");
}
}
fn earliest(&self) -> Option<u64> {
self.marks.lock().unwrap().first().copied()
}
}
pub(crate) struct Mark {
pos: u64,
registry: Arc<MarkRegistry>,
}
impl Mark {
#[inline]
#[allow(dead_code)]
pub fn pos(&self) -> u64 {
self.pos
}
}
impl Drop for Mark {
fn drop(&mut self) {
self.registry.remove(self.pos);
}
}
impl std::fmt::Debug for Mark {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("Mark").field("pos", &self.pos).finish()
}
}
#[derive(Debug)]
pub(crate) struct MarkBuf {
ring: RingBuf,
registry: Arc<MarkRegistry>,
unparsed: u64,
}
#[allow(dead_code)] impl MarkBuf {
pub fn new(capacity: usize) -> Self {
Self {
ring: RingBuf::new(capacity),
registry: MarkRegistry::new(),
unparsed: 0,
}
}
#[inline]
pub fn unparsed(&self) -> u64 {
self.unparsed
}
#[inline]
pub fn end(&self) -> u64 {
self.ring.end()
}
#[inline]
pub fn unparsed_len(&self) -> usize {
(self.ring.end() - self.unparsed) as usize
}
pub fn unparsed_split(&self) -> Split<'_> {
let len = self.unparsed_len();
if len == 0 {
return Split::new(&[], &[]);
}
self.ring.split(self.unparsed, len)
}
pub fn add_mark(&self) -> Mark {
self.add_mark_at(self.unparsed)
}
pub fn add_mark_at(&self, pos: u64) -> Mark {
assert!(
pos >= self.ring.start() && pos <= self.unparsed,
"add_mark_at({pos}): must be within [{}, {}]",
self.ring.start(),
self.unparsed,
);
self.registry.add(pos);
Mark {
pos,
registry: Arc::clone(&self.registry),
}
}
#[inline]
pub fn capacity(&self) -> usize {
self.ring.capacity()
}
#[inline]
pub fn split(&self, pos: u64, len: usize) -> Split<'_> {
self.ring.split(pos, len)
}
pub fn advance_unparsed(&mut self, new_unparsed: u64) {
assert!(
self.unparsed <= new_unparsed && new_unparsed <= self.ring.end(),
"advance_unparsed({new_unparsed}): must be within [{}, {}]",
self.unparsed,
self.ring.end(),
);
self.unparsed = new_unparsed;
}
pub fn spare_capacity(&mut self, reserve: usize) -> (&mut [u8], &mut [u8]) {
self.reclaim();
self.ring.spare_capacity(reserve)
}
pub fn advance_end(&mut self, n: usize) {
self.ring.advance_end(n);
}
fn reclaim(&mut self) {
let floor = self.registry.earliest().unwrap_or(self.unparsed);
debug_assert!(floor <= self.unparsed);
self.ring.advance_to(floor);
}
#[cfg(test)]
pub fn extend(&mut self, data: &[u8]) {
self.reclaim();
self.ring.extend(data);
}
}
pub(crate) struct PacketRef<'a> {
pub meta: crate::rtp::PacketMeta,
buf: &'a MarkBuf,
payload_pos: u64,
payload_len: u16,
}
impl<'a> PacketRef<'a> {
pub fn new(
meta: crate::rtp::PacketMeta,
buf: &'a MarkBuf,
payload_pos: u64,
payload_len: u16,
) -> Self {
Self {
meta,
buf,
payload_pos,
payload_len,
}
}
pub fn mark(&self) -> Mark {
self.buf.add_mark_at(self.payload_pos)
}
#[inline]
pub fn payload_pos(&self) -> u64 {
self.payload_pos
}
#[inline]
pub fn payload_len(&self) -> u16 {
self.payload_len
}
#[inline]
pub fn payload(&self) -> Split<'_> {
self.buf
.split(self.payload_pos, usize::from(self.payload_len))
}
#[inline]
pub fn buf(&self) -> &MarkBuf {
self.buf
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn ring_basic() {
let mut buf = RingBuf::new(8);
assert_eq!(buf.capacity(), 8);
assert!(buf.is_empty());
buf.extend(b"hello");
assert_eq!(buf.len(), 5);
assert_eq!(buf.split(0, 5).slices(), (&b"hello"[..], &b""[..]));
buf.advance_to(3);
assert_eq!(buf.start(), 3);
assert_eq!(buf.len(), 2);
assert_eq!(buf.split(3, 2).slices(), (&b"lo"[..], &b""[..]));
}
#[test]
fn ring_wrap() {
let mut buf = RingBuf::new(8);
buf.extend(b"abcdefgh");
assert_eq!(buf.available(), 0);
buf.advance_to(6);
assert_eq!(buf.available(), 6);
buf.extend(b"ijkl");
assert_eq!(buf.len(), 6);
assert_eq!(buf.split(6, 6).slices(), (&b"gh"[..], &b"ijkl"[..]));
}
#[test]
fn ring_realloc() {
let mut buf = RingBuf::new(4);
buf.extend(b"ab");
buf.advance_to(1);
buf.extend(b"cde");
assert_eq!(buf.len(), 4);
assert_eq!(buf.available(), 0);
buf.extend(b"fg");
assert!(buf.capacity() >= 8);
assert_eq!(buf.len(), 6);
let (s1, s2) = buf.split(1, 6).slices();
let mut all = Vec::new();
all.extend_from_slice(s1);
all.extend_from_slice(s2);
assert_eq!(&all, b"bcdefg");
}
#[test]
fn ring_realloc_wrapped_odd_parity() {
let mut buf = RingBuf::new(8);
buf.extend(b"abcdefgh");
buf.advance_to(8);
buf.extend(b"ABCDEFGH");
buf.advance_to(11); buf.extend(b"xyz"); assert_eq!(buf.len(), 8);
assert_eq!(buf.available(), 0);
buf.extend(b"!"); assert_eq!(buf.len(), 9);
let (s1, s2) = buf.split(11, 9).slices();
let mut all = Vec::new();
all.extend_from_slice(s1);
all.extend_from_slice(s2);
assert_eq!(&all, b"DEFGHxyz!");
}
#[test]
fn ring_realloc_from_every_state() {
const CAP: usize = 8;
for start in 0..(4 * CAP as u64) {
for len in 1..=CAP {
let mut buf = RingBuf::new(CAP);
let mut pos = 0u64;
while pos < start {
let n = ((start - pos) as usize).min(CAP);
buf.extend(&vec![0u8; n]);
pos += n as u64;
buf.advance_to(pos);
}
assert_eq!(buf.capacity(), CAP);
let data: Vec<u8> = (0..len).map(|i| (start as usize + i) as u8).collect();
buf.extend(&data);
let more: Vec<u8> = (0..CAP).map(|i| (i as u8) | 0x80).collect();
buf.extend(&more); assert!(buf.capacity() > CAP, "start={start} len={len}");
let want: Vec<u8> = data.iter().chain(more.iter()).copied().collect();
let (s1, s2) = buf.split(start, want.len()).slices();
let got: Vec<u8> = s1.iter().chain(s2.iter()).copied().collect();
assert_eq!(got, want, "start={start} len={len}");
}
}
}
#[test]
fn spare_capacity_when_full() {
let mut buf = RingBuf::new(8);
buf.extend(b"abcdefgh");
assert_eq!(buf.available(), 0);
let (s1, s2) = buf.spare_capacity(0);
assert_eq!((s1.len(), s2.len()), (0, 0));
}
#[test]
fn spare_capacity_and_advance_end() {
let mut buf = RingBuf::new(8);
let (first, _) = buf.spare_capacity(5);
first[..5].copy_from_slice(b"hello");
buf.advance_end(5);
assert_eq!(buf.len(), 5);
assert_eq!(buf.data_split().slices(), (&b"hello"[..], &b""[..]));
}
#[test]
fn data_split_empty() {
let buf = RingBuf::new(8);
assert_eq!(buf.data_split().slices(), (&b""[..], &b""[..]));
}
#[test]
fn spare_capacity_after_wrap() {
let mut buf = RingBuf::new(8);
let (s1, _) = buf.spare_capacity(7);
s1[..7].copy_from_slice(b"abcdefg");
buf.advance_end(7);
buf.advance_to(buf.end());
let (s1, s2) = buf.spare_capacity(5);
assert!(
s1.len() + s2.len() >= 5,
"s1.len()={}, s2.len()={}",
s1.len(),
s2.len(),
);
}
#[test]
fn mark_raii() {
let mut buf = MarkBuf::new(16);
buf.extend(b"abcdefgh");
buf.advance_unparsed(4);
let mark = buf.add_mark(); assert_eq!(mark.pos(), 4);
buf.advance_unparsed(8);
assert_eq!(buf.split(4, 4).slices(), (&b"efgh"[..], &b""[..]));
drop(mark);
buf.extend(b"more");
}
#[test]
fn mark_multiple() {
let mut buf = MarkBuf::new(16);
buf.extend(b"0123456789");
buf.advance_unparsed(2);
let mark1 = buf.add_mark(); buf.advance_unparsed(5);
let mark2 = buf.add_mark();
assert_eq!(buf.split(2, 3).slices(), (&b"234"[..], &b""[..]));
assert_eq!(buf.split(5, 5).slices(), (&b"56789"[..], &b""[..]));
drop(mark1);
buf.extend(b"");
assert_eq!(buf.split(5, 5).slices(), (&b"56789"[..], &b""[..]));
drop(mark2);
}
#[test]
fn mark_release_order() {
let mut buf = MarkBuf::new(8);
buf.extend(b"abcdef");
buf.advance_unparsed(2);
let mark1 = buf.add_mark();
buf.advance_unparsed(4);
let mark2 = buf.add_mark();
buf.advance_unparsed(6);
let mark3 = buf.add_mark();
drop(mark2);
assert_eq!(buf.split(2, 4).slices(), (&b"cdef"[..], &b""[..]));
drop(mark1);
drop(mark3);
}
#[test]
fn unparsed_split() {
let mut buf = MarkBuf::new(16);
buf.extend(b"hello world");
assert_eq!(buf.unparsed_len(), 11);
assert_eq!(
buf.unparsed_split().slices(),
(&b"hello world"[..], &b""[..])
);
buf.advance_unparsed(6);
assert_eq!(buf.unparsed_len(), 5);
assert_eq!(buf.unparsed_split().slices(), (&b"world"[..], &b""[..]));
}
#[test]
fn mark_buf_growth_like_tcp_reads() {
for mark_round in 0..40u64 {
const READ: usize = 4096;
let mut buf = MarkBuf::new(64 * 1024);
let mut mark = None;
let mut model = Vec::new();
let mut mark_pos = 0;
for round in 0..48u64 {
let pos = buf.end();
let (s1, s2) = buf.spare_capacity(READ);
assert!(s1.len() + s2.len() >= READ);
let mid = READ.min(s1.len());
for (i, b) in s1[..mid].iter_mut().enumerate() {
*b = (pos + i as u64) as u8;
}
for (i, b) in s2[..READ - mid].iter_mut().enumerate() {
*b = (pos + (mid + i) as u64) as u8;
}
buf.advance_end(READ);
buf.advance_unparsed(buf.end());
if round == mark_round {
mark_pos = pos;
mark = Some(buf.add_mark_at(pos));
model.clear();
}
if round >= mark_round {
model.extend((pos..pos + READ as u64).map(|p| p as u8));
}
}
let (s1, s2) = buf.split(mark_pos, model.len()).slices();
let got: Vec<u8> = s1.iter().chain(s2.iter()).copied().collect();
assert_eq!(got, model, "mark_round={mark_round}");
drop(mark);
}
}
}