Skip to main content

moq_net/model/
group.rs

1//! A group is a stream of frames, split into a [Producer] and [Consumer] handle.
2//!
3//! A [Producer] writes an ordered stream of frames.
4//! Frames can be written all at once ([Producer::write_frame]), or in chunks
5//! ([Producer::create_frame]).
6//!
7//! A [Consumer] reads an ordered stream of frames.
8//! The reader can be cloned, in which case each reader receives a copy of each frame. (fanout)
9//!
10//! Frames are numbered from 0 in write order. A group can be short at its front or its
11//! back but never in the middle: [Producer::start_at] starts it later, so a handle can
12//! carry the tail of a group whose leading frames came from somewhere else, and
13//! [Producer::finish] ends it wherever writing stopped. [Consumer::set_frames] bounds a reader to a sub-range the same way [`track::Subscriber`]
14//! bounds group sequences.
15//!
16//! The stream is closed with [Error] when all writers or readers are dropped.
17use crate::cache;
18use crate::frame::{self, Frame, FrameBuf};
19use crate::{Cap, Timescale, stats, track};
20use std::collections::VecDeque;
21use std::mem::MaybeUninit;
22use std::ops::{Bound, RangeBounds};
23use std::sync::Arc;
24use std::sync::atomic::{AtomicBool, Ordering};
25use std::task::{Poll, ready};
26
27use crate::{Error, IntoBytes, Result, Timestamp};
28
29/// Maximum total size of frames in a group.
30///
31/// A write that would exceed this aborts the group with [`Error::GroupTooLarge`].
32/// Doubles as the per-frame size cap: a larger declared size is [`Error::FrameTooLarge`]
33/// before allocating, so one maximum-size frame can fill a group.
34pub const MAX_CACHE_BYTES: u64 = 32 * 1024 * 1024; // 32 MB
35
36/// Maximum number of frames in a group.
37///
38/// 8192 is the largest legal group; the 8193rd write returns [`Error::GroupTooLarge`]
39/// and aborts the group.
40pub const MAX_GROUP_FRAMES: usize = 8192;
41
42/// Slots `VecDeque` rounds a group's first frame up to.
43///
44/// A `RawVec` detail rather than a knob, so it is asserted rather than trusted: std
45/// handing out more would silently undercharge every cached group.
46const FRAME_SLOTS: usize = 4;
47
48/// Heap one cached group costs beyond its frame payloads, excluding the track-side
49/// bookkeeping in [`track::CACHE_OVERHEAD`].
50///
51/// A group is one kio channel (allocated whether or not anything ever parks on it), the
52/// `Arc<Alive>` its producer clones share, and the frame slots the first write rounds up
53/// to. Half of [`cache::ENTRY_OVERHEAD`]; see it for why this is derived rather than
54/// measured.
55pub(crate) const CACHE_OVERHEAD: u64 = (kio::Producer::<GroupState>::HEAP
56	// `Alive` behind an `Arc`'s two reference counts, which it is pointer-aligned to sit
57	// straight after.
58	+ 2 * size_of::<usize>()
59	+ size_of::<Alive>()
60	+ FRAME_SLOTS * size_of::<Frame>()) as u64;
61
62/// A group contains a sequence number because they can arrive out of order.
63///
64/// You can use [track::Producer::append_group] if you just want to +1 the sequence number.
65#[derive(Clone, Copy, Debug, Hash, Eq, PartialEq, Ord, PartialOrd)]
66pub struct Info {
67	/// Per-track sequence number used to detect ordering and gaps. Higher numbers
68	/// supersede lower ones; consumers may skip late arrivals.
69	pub sequence: u64,
70}
71
72impl Info {
73	/// Create an untimed producer for this group.
74	///
75	/// Test-only: real groups are created via [`track::Producer`], which
76	/// supplies the parent track's [`track::Info`]. This helper exists for in-crate
77	/// tests that don't exercise timestamps.
78	#[cfg(test)]
79	pub(crate) fn produce(self) -> Producer {
80		Producer::new(self, track::Info::default(), Default::default())
81	}
82}
83
84impl From<usize> for Info {
85	fn from(sequence: usize) -> Self {
86		Self {
87			sequence: sequence as u64,
88		}
89	}
90}
91
92impl From<u64> for Info {
93	fn from(sequence: u64) -> Self {
94		Self { sequence }
95	}
96}
97
98impl From<u32> for Info {
99	fn from(sequence: u32) -> Self {
100		Self {
101			sequence: sequence as u64,
102		}
103	}
104}
105
106impl From<u16> for Info {
107	fn from(sequence: u16) -> Self {
108		Self {
109			sequence: sequence as u64,
110		}
111	}
112}
113
114/// The in-flight (tail) frame being written. At most one exists at a time, since a
115/// group is a single ordered stream.
116pub(crate) struct Partial {
117	timestamp: Timestamp,
118	buf: FrameBuf,
119	// How much of `buf` has been charged to the cache so far.
120	charged: u64,
121}
122
123/// Shared group state. `pub(crate)` so [`frame`] handles can observe the abort flag
124/// while streaming a partial frame.
125#[derive(Default)]
126pub(crate) struct GroupState {
127	// Completed frames, each a contiguous payload. `offset` is the first frame this
128	// handle holds, raised by [`Producer::start_at`].
129	pub(crate) frames: VecDeque<Frame>,
130
131	// The single in-flight frame, if one is open.
132	pub(crate) partial: Option<Partial>,
133
134	// Index of the first frame this handle holds: any the group deliberately started
135	// past (see [`Producer::start_at`]). Reading below it is [`Error::Lagged`]; the
136	// frames are not here.
137	pub(crate) offset: usize,
138
139	// The index the next frame written will get. Tracked separately from `frames` so it
140	// survives the cache being released: a route taking the track over needs to know where
141	// production stopped, and an abort is exactly when it asks.
142	next_index: usize,
143
144	// One past the last frame that was fully written. Trails `next_index` while a chunked
145	// frame is in flight, which is the frame a replacement route has to redeliver: only
146	// its opener saw the payload, and only partly.
147	committed: usize,
148
149	// The total size (in bytes) of all cached frames plus what the in-flight frame has
150	// written so far.
151	pub(crate) cache: u64,
152
153	// Mirrors `cache` into the track's shared cache pool, so the group's bytes count
154	// against the byte budget tracks evict toward.
155	charge: cache::Charge,
156
157	// The first frame's timestamp, recorded once and never revised: the group's
158	// presentation start. Kept here rather than read off `frames` so an abort
159	// doesn't erase where the group sat in time. `None` until the first frame is
160	// written, which is the only honest answer: an empty group has not presented
161	// anything yet.
162	timestamp: Option<Timestamp>,
163
164	// The newest frame's timestamp: the group's presentation end so far. A reader
165	// that has taken every frame sits here, which is what a drift budget measures it
166	// against. Kept alongside `timestamp` for the same reasons.
167	latest: Option<Timestamp>,
168
169	// Once finalized, the total number of frames the group will ever contain. Recorded
170	// at finish so the count outlives an abort that clears the cache.
171	pub(crate) fin: Option<usize>,
172
173	// The error that caused the group to be aborted, if any. Mirrored into
174	// `Alive::aborted`, so [`Producer::abort`] stays the only writer: anything else
175	// setting this would leave track scans reading a group as live.
176	pub(crate) abort: Option<Error>,
177}
178
179impl GroupState {
180	/// Content still available to a reader of this group.
181	///
182	/// Counts the in-flight frame at its declared size, like [`Self::content_range`]: a
183	/// reader that skips it misses the whole frame, however much has arrived.
184	fn content(&self) -> stats::Content {
185		let unwritten = self
186			.partial
187			.as_ref()
188			.map_or(0, |partial| partial.buf.size() as u64 - partial.charged);
189		stats::Content {
190			bytes: self.cache + unwritten,
191			frames: self.next_index.saturating_sub(self.offset) as u64,
192			groups: 1,
193			datagrams: 0,
194		}
195	}
196
197	/// Content in the half-open frame range that is still cached here.
198	pub(crate) fn content_range(&self, start: usize, end: usize) -> stats::Content {
199		let start = start.max(self.offset);
200		let end = end.min(self.next_index);
201		if start >= end {
202			return stats::Content::default();
203		}
204
205		let local_start = start.saturating_sub(self.offset).min(self.frames.len());
206		let local_end = end.saturating_sub(self.offset).min(self.frames.len());
207		let mut bytes = self
208			.frames
209			.range(local_start..local_end)
210			.map(|frame| frame.payload.len() as u64)
211			.sum();
212		if start <= self.committed
213			&& self.committed < end
214			&& let Some(partial) = &self.partial
215		{
216			bytes += partial.buf.size() as u64;
217		}
218
219		stats::Content {
220			bytes,
221			frames: (end - start) as u64,
222			groups: 0,
223			datagrams: 0,
224		}
225	}
226
227	/// Resolve the source for the frame at `index`: a completed frame (whole) or the
228	/// in-flight tail (streamed). Used by [`Consumer::poll_next_frame`].
229	fn poll_frame_source(&self, index: usize) -> Poll<Result<Option<(frame::Info, frame::Source)>>> {
230		if index < self.offset {
231			return Poll::Ready(Err(Error::Lagged));
232		}
233		let local = index - self.offset;
234		if let Some(f) = self.frames.get(local) {
235			// A frame read is a cache access: stamp it so expiry and the eviction
236			// walk spare a group a consumer is actively draining.
237			self.charge.refresh();
238			let info = frame::Info {
239				size: f.payload.len() as u64,
240				timestamp: f.timestamp,
241			};
242			return Poll::Ready(Ok(Some((info, frame::Source::Complete(f.payload.clone())))));
243		}
244		if local == self.frames.len()
245			&& let Some(p) = &self.partial
246		{
247			self.charge.refresh();
248			let info = frame::Info {
249				size: p.buf.size() as u64,
250				timestamp: p.timestamp,
251			};
252			return Poll::Ready(Ok(Some((info, frame::Source::Partial(p.buf.clone())))));
253		}
254		ready!(self.poll_terminal(index))?;
255		Poll::Ready(Ok(None))
256	}
257
258	/// Resolve the group's terminal state for a reader positioned at `index`.
259	///
260	/// A finished group is still aborted once its frames are released to free memory
261	/// (aged out of the track's max age window, or evicted by the cache pool). A reader
262	/// that already consumed every frame is missing nothing, so it gets the clean end of
263	/// group; one that fell short sees the abort rather than a silently truncated stream.
264	fn poll_terminal(&self, index: usize) -> Poll<Result<()>> {
265		match (self.fin, &self.abort) {
266			(Some(total), Some(err)) if index < total => Poll::Ready(Err(err.clone())),
267			(Some(_), _) => Poll::Ready(Ok(())),
268			(None, Some(err)) => Poll::Ready(Err(err.clone())),
269			(None, None) => Poll::Pending,
270		}
271	}
272
273	/// Resolve whether a reader at `index` can still make progress, answering the same
274	/// question as a read without consuming anything.
275	fn poll_end(&self, index: usize) -> Poll<Result<()>> {
276		if index < self.offset {
277			return Poll::Ready(Err(Error::Lagged));
278		}
279		self.poll_terminal(index)
280	}
281
282	/// Record where the group starts and currently ends in presentation time.
283	/// `timestamp` keeps the first frame only; `latest` follows every frame.
284	fn stamp(&mut self, timestamp: Timestamp) {
285		self.timestamp.get_or_insert(timestamp);
286		self.latest = Some(timestamp);
287	}
288
289	/// Whether adding `extra_frames` totaling `extra_bytes` would exceed the group budget.
290	fn would_overflow(&self, extra_frames: usize, extra_bytes: u64) -> bool {
291		self.next_index.saturating_sub(self.offset).saturating_add(extra_frames) > MAX_GROUP_FRAMES
292			|| self.cache.saturating_add(extra_bytes) > MAX_CACHE_BYTES
293	}
294
295	/// Charge the in-flight frame's bytes written since its last charge, as a write access.
296	///
297	/// Returns the coarse tick it stamped, like [`cache::Charge::add`].
298	fn charge_partial(&mut self) -> Option<u64> {
299		let written = match &mut self.partial {
300			Some(partial) => {
301				let written = partial.buf.written(Ordering::Acquire) as u64;
302				written - std::mem::replace(&mut partial.charged, written)
303			}
304			None => 0,
305		};
306		self.cache += written;
307		self.charge.add(written)
308	}
309
310	/// Drop the cached frames (and any in-flight tail) and release their pool charge.
311	fn release(&mut self) {
312		self.frames.clear();
313		self.partial = None;
314		self.cache = 0;
315		self.charge.clear();
316	}
317}
318
319fn modify(state: &kio::Producer<GroupState>) -> Result<kio::Mut<'_, GroupState>> {
320	state.write().map_err(|r| r.abort.clone().unwrap_or(Error::Dropped))
321}
322
323/// Writes frames to a group in order.
324///
325/// Each group is delivered independently over a QUIC stream.
326/// Use [Self::write_frame] for simple single-buffer frames,
327/// or [Self::create_frame] for multi-chunk streaming writes.
328pub struct Producer {
329	// Mutable stream state.
330	state: kio::Producer<GroupState>,
331
332	// The group header containing the sequence number. A small `Copy` value,
333	// inherited by each frame (see [`Self::create_frame`]).
334	info: Info,
335
336	// The parent track's properties, inherited rather than passed piecemeal. Its
337	// `timescale` is used by [`Self::create_frame`] to normalize every frame's
338	// timestamp into the track scale before it enters the stream. Threaded down by
339	// value from [`track::Producer::create_group`] / `append_group`.
340	track: track::Info,
341
342	// The parent track's account against the shared cache pool. Held here as well as
343	// in the group's `cache::Charge` so a frame write can settle the track's eviction
344	// debt with the group lock released.
345	cache: Arc<cache::Track>,
346
347	// Ingress payload meter, set by a tagged [`track::Producer`] via
348	// [`Self::with_meter`]. Empty (no-op) for an untagged group.
349	stats: stats::Meter,
350
351	// Shared by every clone: its `Drop` is the abrupt-teardown, running exactly once
352	// when the last of them goes.
353	alive: Arc<Alive>,
354}
355
356/// Ends the group when the last [`Producer`] clone drops, including the clone the
357/// parent track holds in its cache.
358///
359/// A refcount rather than a "am I the last one?" check inside `Drop`: that answer is
360/// a snapshot, and acting on it is exactly what can invalidate it. Holding a producer
361/// of its own also keeps the state writable until the teardown has run, whatever order
362/// the last owner's fields drop in.
363struct Alive {
364	info: Info,
365	state: kio::Producer<GroupState>,
366	// Monotone mirror of `GroupState::abort` for track scans that already hold the
367	// track lock. A stale false only hands out a group that is concurrently aborting;
368	// true is stored after the abort exists, so it can never hide a live group.
369	//
370	// Only ever read on its own. A decision that pairs the abort with something else
371	// out of `GroupState` has to read both under one guard, or the two halves can
372	// straddle the abort: see `Producer::live_first_frame`.
373	aborted: AtomicBool,
374	// The cache stamp `GroupState::charge` maintains, held here as well so the
375	// eviction and expiry walks can weigh a candidate without taking the group lock
376	// they already hold the track lock over.
377	access: Arc<cache::Access>,
378}
379
380impl Drop for Alive {
381	fn drop(&mut self) {
382		// See track::Alive: the last producer dropping without a clean finish releases
383		// the cached frames so a stale consumer can't pin their buffers forever. A
384		// finished group keeps its cache so consumers can drain.
385		//
386		// Check Ok and Err: Ok is unreachable after a deliberate close.
387		match self.state.write() {
388			Ok(mut state) => {
389				if state.fin.is_some() || state.abort.is_some() {
390					return;
391				}
392				tracing::warn!(
393					sequence = self.info.sequence,
394					"group::Producer dropped without finish() or abort()"
395				);
396				state.release();
397			}
398			Err(state) => {
399				if state.fin.is_some() || state.abort.is_some() {
400					return;
401				}
402				tracing::warn!(
403					sequence = self.info.sequence,
404					"group::Producer dropped without finish() or abort()"
405				);
406			}
407		}
408	}
409}
410
411impl std::ops::Deref for Producer {
412	type Target = Info;
413
414	fn deref(&self) -> &Self::Target {
415		&self.info
416	}
417}
418
419impl Producer {
420	/// Create a group producer bound to its parent track's [`track::Info`] and cache
421	/// account.
422	///
423	/// Crate-private: groups are only constructed via [`track::Producer`], which
424	/// threads both down so properties like the timescale are inherited rather than
425	/// passed in. Every frame added to this group is normalized to the track's
426	/// timescale by [`Self::create_frame`].
427	///
428	/// Charges the group into `cache`, so its cached bytes count against the budget the
429	/// track evicts toward under memory pressure.
430	pub(crate) fn new(info: Info, track: track::Info, cache: Arc<cache::Track>) -> Self {
431		let state = kio::Producer::<GroupState>::default();
432		let charge = cache.charge();
433		let access = charge.access();
434		state.write().ok().expect("a new group is open").charge = charge;
435		let alive = Arc::new(Alive {
436			info,
437			state: state.clone(),
438			aborted: AtomicBool::new(false),
439			access,
440		});
441		Self {
442			info,
443			state,
444			track,
445			cache,
446			stats: stats::Meter::default(),
447			alive,
448		}
449	}
450
451	/// Attach an ingress payload meter, counting this as one delivered group.
452	/// Called by a tagged [`track::Producer`] when it creates the group.
453	pub(crate) fn with_meter(mut self, meter: stats::Meter) -> Self {
454		meter.group();
455		self.stats = meter;
456		self
457	}
458
459	/// The group header.
460	pub(crate) fn info(&self) -> Info {
461		self.info
462	}
463
464	/// The parent track's timescale.
465	pub fn timescale(&self) -> Timescale {
466		self.track.timescale
467	}
468
469	/// Start the group at frame `index` rather than 0, so the first frame written lands
470	/// there.
471	///
472	/// A group can be short at its front or its back, never in the middle: this trims
473	/// the front, and simply stopping (then [`finish`](Self::finish)ing) trims the back.
474	/// The frames below `index` are not a gap this handle will ever fill, so a reader
475	/// positioned below it gets [`Error::Lagged`]. They belong to whoever produced the
476	/// head of the group, typically another route serving the same track (see
477	/// [`crate::track::Subscriber`]).
478	///
479	/// The counterpart of [`Consumer::set_frames`], which positions a *reader* the same
480	/// way. Where the group begins is part of its shape, so this must come before the
481	/// first frame; afterwards it returns [`Error::Closed`].
482	pub fn start_at(&mut self, index: u64) -> Result<()> {
483		let index = usize::try_from(index).map_err(|_| Error::BoundsExceeded(crate::coding::BoundsExceeded))?;
484		if index == usize::MAX {
485			return Err(Error::BoundsExceeded(crate::coding::BoundsExceeded));
486		}
487
488		let mut state = modify(&self.state)?;
489		// Every write advances `next_index` past `offset`, so this is "nothing written
490		// yet".
491		if state.fin.is_some() || state.next_index != state.offset {
492			return Err(Error::Closed);
493		}
494		state.offset = index;
495		state.next_index = index;
496		state.committed = index;
497		Ok(())
498	}
499
500	/// A helper method to write a frame from a single byte buffer.
501	///
502	/// If you want to write multiple chunks, use [Self::create_frame] to get a frame producer.
503	/// But an upfront size is required.
504	///
505	/// `timestamp` is converted into the parent track's timescale. For data without
506	/// a presentation time, pass [`Timestamp::now`] explicitly.
507	pub fn write_frame<B: IntoBytes>(&mut self, timestamp: Timestamp, data: B) -> Result<()> {
508		let timestamp = timestamp
509			.convert(self.track.timescale)
510			.map_err(|_| Error::TimestampMismatch)?;
511		let payload = data.into_bytes();
512		if payload.len() as u64 > MAX_CACHE_BYTES {
513			return Err(Error::FrameTooLarge);
514		}
515
516		let mut state = modify(&self.state)?;
517		if state.fin.is_some() {
518			return Err(Error::Closed);
519		}
520		if state.partial.is_some() {
521			return Err(Error::FrameOpen);
522		}
523		let next_index = state
524			.next_index
525			.checked_add(1)
526			.ok_or(Error::BoundsExceeded(crate::coding::BoundsExceeded))?;
527		debug_assert!(state.partial.is_none(), "a frame is already open");
528		let size = payload.len() as u64;
529		if state.would_overflow(1, size) {
530			return Err(self.abort_too_large(state));
531		}
532		state.cache += size;
533		let now = state.charge.add(size);
534		state.frames.push_back(Frame { timestamp, payload });
535		state.next_index = next_index;
536		state.committed = state.next_index;
537		state.stamp(timestamp);
538		drop(state);
539
540		// With the group lock released (lock order is track then group), settle
541		// eviction debt if enough has been written since the track last paid.
542		self.cache.settle(now);
543
544		// Ingress payload: one whole frame written.
545		self.stats.frames(1);
546		self.stats.bytes(size);
547		Ok(())
548	}
549
550	/// Write a whole batch of frames at once, draining `frames`.
551	///
552	/// One lock covers the batch, so an ingest with several frames in hand pays the
553	/// group mutex and the track's eviction settle once rather than per frame. Build
554	/// the batch with [`frame::Buffer::push`].
555	///
556	/// The batch is validated before anything is written, so a rejected frame leaves
557	/// both the group and the buffer exactly as they were, ready to retry or redirect.
558	/// Returns [`Error::FrameOpen`] if another handle is streaming a frame into this
559	/// group, since appending around it would reorder the group.
560	pub fn write_frames<const N: usize>(&mut self, frames: &mut frame::Buffer<N>) -> Result<()> {
561		for frame in frames.filled() {
562			frame
563				.timestamp
564				.convert(self.track.timescale)
565				.map_err(|_| Error::TimestampMismatch)?;
566			if frame.payload.len() as u64 > MAX_CACHE_BYTES {
567				return Err(Error::FrameTooLarge);
568			}
569		}
570
571		let count = frames.len();
572		let bytes: u64 = frames.filled().iter().map(|frame| frame.payload.len() as u64).sum();
573		let mut state = modify(&self.state)?;
574		if state.fin.is_some() {
575			return Err(Error::Closed);
576		}
577		if state.partial.is_some() {
578			return Err(Error::FrameOpen);
579		}
580		let next_index = state
581			.next_index
582			.checked_add(count)
583			.ok_or(Error::BoundsExceeded(crate::coding::BoundsExceeded))?;
584		if state.would_overflow(count, bytes) {
585			return Err(self.abort_too_large(state));
586		}
587
588		// The last frame's tick, reused below so settling does not re-read the clock.
589		let mut now = None;
590		for mut frame in frames.drain() {
591			frame.timestamp = frame
592				.timestamp
593				.convert(self.track.timescale)
594				.expect("timestamp scale checked above");
595			let size = frame.payload.len() as u64;
596			state.cache += size;
597			now = state.charge.add(size);
598			state.stamp(frame.timestamp);
599			state.frames.push_back(frame);
600		}
601		state.next_index = next_index;
602		state.committed = next_index;
603		drop(state);
604
605		self.cache.settle(now);
606		self.stats.frames(count as u64);
607		self.stats.bytes(bytes);
608		Ok(())
609	}
610
611	/// Create a frame with an upfront size and presentation timestamp, streamed in
612	/// chunks. Borrows the group exclusively until the returned [`frame::Producer`]
613	/// is finished or dropped, so only one frame is open at a time.
614	///
615	/// The `timestamp` is converted into the parent track's timescale, so the scale you
616	/// build it with doesn't have to match the track. Returns [`Error::FrameTooLarge`]
617	/// if the declared size exceeds the group's byte budget (refused before allocating)
618	/// or [`Error::TimestampMismatch`] if the timestamp can't be converted (overflow).
619	pub fn create_frame(&mut self, frame: frame::Info) -> Result<frame::Producer<'_>> {
620		// The buffer allocates on its first write, so refusing the frame costs nothing.
621		let buf = FrameBuf::new(frame.size as usize);
622		let info = self.open_frame(frame, &buf)?;
623		let meter = self.stats.clone();
624		Ok(frame::Producer::new(self, buf, info).with_meter(meter))
625	}
626
627	/// The owned counterpart of [`Self::create_frame`], for the wire drivers that
628	/// stream a frame across polls and cannot hold the group borrowed inside their
629	/// state. The one-live-frame rule the borrow normally enforces becomes the
630	/// caller's promise; see [`frame::ProducerOwned`].
631	///
632	/// The declared size is the peer's claim, so the buffer is allocated up front only
633	/// while it fits the session's `budget`; otherwise it grows with the bytes received.
634	pub(crate) fn create_frame_owned(
635		&mut self,
636		frame: frame::Info,
637		budget: &frame::Budget,
638	) -> Result<frame::ProducerOwned> {
639		let size = frame.size as usize;
640		let reserved = budget.reserve(size);
641		let buf = match reserved {
642			Some(_) => FrameBuf::new(size),
643			None => FrameBuf::growing(size),
644		};
645		let info = self.open_frame(frame, &buf)?;
646		let meter = self.stats.clone();
647		Ok(frame::ProducerOwned::new(self.clone(), buf, info, reserved).with_meter(meter))
648	}
649
650	/// Open `buf` as the in-flight frame, returning its header in the track's timescale.
651	///
652	/// Its bytes are charged to the cache as they are written, not here: the declared
653	/// size is only a promise until they arrive.
654	fn open_frame(&mut self, frame: frame::Info, buf: &FrameBuf) -> Result<frame::Info> {
655		let timestamp = frame
656			.timestamp
657			.convert(self.track.timescale)
658			.map_err(|_| Error::TimestampMismatch)?;
659		if frame.size > MAX_CACHE_BYTES {
660			return Err(Error::FrameTooLarge);
661		}
662
663		let mut state = modify(&self.state)?;
664		if state.fin.is_some() {
665			return Err(Error::Closed);
666		}
667		if state.partial.is_some() {
668			return Err(Error::FrameOpen);
669		}
670		let next_index = state
671			.next_index
672			.checked_add(1)
673			.ok_or(Error::BoundsExceeded(crate::coding::BoundsExceeded))?;
674		// Only one frame is ever in flight, so the declared size is the most the cache
675		// can grow by before the next check.
676		if state.would_overflow(1, frame.size) {
677			return Err(self.abort_too_large(state));
678		}
679		let now = state.charge.record_write();
680		state.partial = Some(Partial {
681			timestamp,
682			buf: buf.clone(),
683			charged: 0,
684		});
685		state.next_index = next_index;
686		// Opening the frame is enough: the header carries the timestamp, so the group's
687		// place in time is known before a single payload byte streams in.
688		state.stamp(timestamp);
689		drop(state);
690
691		// With the group lock released (lock order is track then group), settle
692		// eviction debt if enough has been written since the track last paid.
693		self.cache.settle(now);
694
695		// Ingress payload: one frame opened; its bytes are counted per chunk as the
696		// producer writes them.
697		self.stats.frames(1);
698
699		Ok(frame::Info {
700			size: frame.size,
701			timestamp,
702		})
703	}
704
705	/// Wake consumers parked on the group channel (called after a partial write).
706	pub(crate) fn frame_notify(&self) {
707		// The chunk that was just written is charged, and is a write access: restart the
708		// retention clock so a straggler group streaming a large frame isn't expired
709		// mid-write. `charge_partial` takes `&mut`, which marks the guard modified: kio
710		// only notifies on a mutably-accessed guard's release, and that notify is what
711		// delivers the chunk to parked readers.
712		let now = self.state.write().ok().and_then(|mut state| state.charge_partial());
713		// A long streamed frame also counts as track activity for the independent
714		// expiry time gate.
715		self.cache.settle(now);
716	}
717
718	/// Commit the in-flight frame as a completed frame (called by [`frame::Producer::finish`]).
719	pub(crate) fn frame_commit(&mut self, frame: Frame) -> Result<()> {
720		let mut state = modify(&self.state)?;
721		// Completing the frame charges whatever it wrote since the last notify, and is a
722		// write access like any chunk, and the only one the payload is guaranteed to get:
723		// the wire ingest defers its chunk notifications to the poll boundary, so a tail
724		// that arrives and completes in one turn never reaches [`Self::frame_notify`].
725		// Without this, a group whose payload streamed in across an idle gap would expire
726		// the instant it finished.
727		let now = state.charge_partial();
728		state.partial = None;
729		state.frames.push_back(frame);
730		state.committed = state.next_index;
731		drop(state);
732
733		// With the group lock released (lock order is track then group), settle
734		// eviction debt and age idle content out, reusing the tick above.
735		self.cache.settle(now);
736		Ok(())
737	}
738
739	/// Fail the group because an in-flight frame couldn't complete (called by
740	/// [`frame::Producer::abort`] / its drop).
741	pub(crate) fn frame_abort(&mut self, err: Error) {
742		let _ = self.clone().abort(err);
743	}
744
745	/// One past the index of the last frame written (completed or in-flight), which is
746	/// also the index the next frame will get.
747	///
748	/// Counts any frames the group [started past](Self::start_at), so it's the group's
749	/// logical length rather than the number of frames this handle holds.
750	pub fn frame_count(&self) -> usize {
751		self.state.read().next_index
752	}
753
754	/// Mark the group as complete; no more frames will be written.
755	///
756	/// Borrows rather than consumes, so a later failure can still be reported through
757	/// [`abort`](Self::abort). The handle also keeps the cached frames readable.
758	pub fn finish(&self) -> Result<()> {
759		let mut state = modify(&self.state)?;
760		if state.partial.is_some() {
761			return Err(Error::FrameOpen);
762		}
763		state.fin = Some(state.next_index);
764		Ok(())
765	}
766
767	/// Abort the group with the given error.
768	///
769	/// Consumes the handle. Drops the cached frames so a stale [`Consumer`] can't pin
770	/// their buffers in memory forever; consumers that haven't drained yet surface the
771	/// abort error instead of the leftover cache.
772	pub fn abort(self, err: Error) -> Result<()> {
773		self.close_aborted(err)
774	}
775
776	/// Abort with `err` unless another handle still owns the group, so a retained clone
777	/// lets go without the unfinished-drop warning and never cuts off a live writer.
778	/// Sound without a lock: only an existing handle can mint another.
779	pub(crate) fn abort_if_last(&self, err: Error) -> Result<()> {
780		if Arc::strong_count(&self.alive) > 1 {
781			return Ok(());
782		}
783		self.close_aborted(err)
784	}
785
786	/// Abort with `err` only while nothing consumes the group, returning whether it is
787	/// closed. Consumer creation and the check share a lock, so a reader arriving after
788	/// [`poll_unused`](Self::poll_unused) keeps the group alive instead of reading the abort.
789	pub(crate) fn abort_unused(&self, err: Error) -> bool {
790		match self.state.write_unused() {
791			kio::Unused::Idle(guard) => {
792				self.commit_abort(guard, err);
793				true
794			}
795			kio::Unused::Closed => true,
796			kio::Unused::Used => false,
797		}
798	}
799
800	fn close_aborted(&self, err: Error) -> Result<()> {
801		self.commit_abort(modify(&self.state)?, err);
802		Ok(())
803	}
804
805	fn commit_abort(&self, mut guard: kio::Mut<'_, GroupState>, err: Error) {
806		guard.abort = Some(err);
807		self.alive.aborted.store(true, Ordering::Release);
808		guard.release();
809		guard.close();
810	}
811
812	/// Abort a write that would grow the group past its budget, holding the lock already
813	/// taken for that write so nothing else lands in between.
814	fn abort_too_large(&self, mut state: kio::Mut<'_, GroupState>) -> Error {
815		let err = Error::GroupTooLarge;
816		state.abort = Some(err.clone());
817		self.alive.aborted.store(true, Ordering::Release);
818		state.release();
819		state.close();
820		err
821	}
822
823	/// Whether the group has been aborted (including pool eviction). The track's
824	/// read paths treat an aborted cached group as absent.
825	///
826	/// Reads the mirror rather than the group's state, so a track scan holding the
827	/// track lock never takes the group's. Monotone, and only ever conservative: a
828	/// concurrent abort can still read as live for the length of [`Self::abort`],
829	/// which hands out a group whose consumer then surfaces the abort.
830	pub(crate) fn is_aborted(&self) -> bool {
831		self.alive.aborted.load(Ordering::Acquire)
832	}
833
834	/// Whether the group was finished: it holds every frame it will ever have.
835	pub(crate) fn is_finished(&self) -> bool {
836		self.state.read().fin.is_some()
837	}
838
839	/// The index of the first frame this group still holds, or `None` once it has been
840	/// aborted. Non-zero when the group started later (see [`Self::start_at`]); a reader
841	/// positioned below it is [`Error::Lagged`].
842	///
843	/// One guard for both halves, deliberately. The track asks this to decide whether a
844	/// cached slot can still answer a request, and reading the abort and the offset
845	/// separately lets the abort land between them: the slot reads live, then hands
846	/// back an offset it only has because it is dead. The mirror
847	/// ([`Self::is_aborted`]) is for scans that ask about the abort alone.
848	pub(crate) fn live_first_frame(&self) -> Option<usize> {
849		let state = self.state.read();
850		state.abort.is_none().then_some(state.offset)
851	}
852
853	/// One past the last frame committed to an unfinished group, when that is past its
854	/// first: where a replacement route resumes. `None` once the group is finished, or
855	/// while it holds nothing a replacement could splice onto.
856	///
857	/// The *committed* count, not the written one: a route dying midway through a
858	/// chunked frame leaves that frame unusable, so the replacement has to send it
859	/// again rather than start after it. Answered under one guard so the count can't be
860	/// weighed against an offset from a different moment.
861	///
862	/// An aborted group still answers: readers that already consumed its head want the
863	/// tail, and the count outlives the released cache.
864	pub(crate) fn resume_frame(&self) -> Option<usize> {
865		let state = self.state.read();
866		if state.fin.is_some() {
867			return None;
868		}
869		(state.committed > state.offset).then_some(state.committed)
870	}
871
872	/// Where the group starts in presentation time: its first frame's timestamp,
873	/// or `None` while no frame has been opened.
874	///
875	/// Stamped once, when the group's first frame arrives, so it measures the group's
876	/// place in the media timeline rather than when it happened to be delivered. That
877	/// is what lets the track tell a burst of old content apart from live content (see
878	/// [`track::Subscriber`]). On protocols whose wire can't carry a timestamp the
879	/// receiver stamps frames with [`Timestamp::now`], which makes this the local
880	/// receive time instead: an estimate that a burst compresses.
881	pub(crate) fn timestamp(&self) -> Option<Timestamp> {
882		self.state.read().timestamp
883	}
884
885	/// Where the group ends in presentation time: its newest frame's timestamp, or
886	/// `None` while no frame has been opened.
887	///
888	/// This is what a drift budget measures an untouched group against. A group is not
889	/// late because it *started* long ago: a two-second group whose tail is level with
890	/// the live edge still has content nobody has read. Only once its newest frame has
891	/// fallen behind is there nothing left worth delivering. Grows as the group does, so
892	/// a group still receiving frames stays fresh and a stalled one ages in place.
893	pub(crate) fn latest(&self) -> Option<Timestamp> {
894		self.state.read().latest
895	}
896
897	/// The group's full cached footprint (payload plus fixed overhead), used by the
898	/// track to size this group as an eviction victim.
899	pub(crate) fn cache_size(&self) -> u64 {
900		self.state.read().charge.size()
901	}
902
903	/// Tick of the group's last cache access, driving eviction protection and age
904	/// expiry (see [`cache::Pool::average`]).
905	pub(crate) fn cache_accessed(&self) -> u64 {
906		self.alive.access.get()
907	}
908
909	/// Coarse clock tick of the group's last cache access, used by age expiry.
910	pub(crate) fn cache_accessed_tick(&self, now: Option<u64>) -> Option<u64> {
911		self.alive.access.tick(now)
912	}
913
914	/// Hold the group as a live track's latest until the guard drops.
915	pub(crate) fn cache_protect(&self) -> cache::Protection {
916		cache::Protection::new(&self.alive.access)
917	}
918
919	/// Whether a live track holds the group as its latest.
920	pub(crate) fn cache_protected(&self) -> bool {
921		self.alive.access.is_protected()
922	}
923
924	/// Enter the group into the evictable population: demoted from the live edge,
925	/// or inserted behind it. Idempotent; a no-op once the group is closed.
926	pub(crate) fn cache_demote(&self) {
927		if let Ok(mut state) = self.state.write() {
928			state.charge.demote();
929		}
930	}
931
932	/// Record a cache access (delivery to a subscriber, a FETCH hit, or a fetched
933	/// backfill's birth), protecting the group from eviction and restarting its
934	/// expiry clock. Stamps through a read guard, whose release never notifies, so
935	/// delivery can't wake every consumer parked on the group. Harmless on a
936	/// closed group: its charge is already cleared.
937	pub(crate) fn cache_refresh(&self) {
938		self.state.read().charge.refresh();
939	}
940
941	/// Create a new consumer for the group.
942	pub fn consume(&self) -> Consumer {
943		self.consumer(self.state.consume())
944	}
945
946	/// Create a consumer, or `None` once the group is aborted. Paired with
947	/// [`abort_unused`](Self::abort_unused): the closed check and the count share its
948	/// lock, so a consumer either exists in time to decline the abort or is never made.
949	pub(crate) fn try_consume(&self) -> Option<Consumer> {
950		self.state.weak().try_consume().map(|state| self.consumer(state))
951	}
952
953	fn consumer(&self, state: kio::Consumer<GroupState>) -> Consumer {
954		Consumer {
955			info: self.info,
956			track: self.track.clone(),
957			inner: ConsumerKind::Plain(Plain {
958				state,
959				index: 0,
960				end: None,
961				prefetch: Prefetch::default(),
962				cache: self.cache.clone(),
963				access: self.alive.access.clone(),
964				refreshed: self.cache.pool().now(),
965			}),
966			// Untagged: a tagged track attaches the egress meter via `with_meter`
967			// when it hands the consumer to a subscriber/fetch.
968			stats: stats::Meter::default(),
969			stale_stats: stats::Meter::default(),
970			expiry: None,
971			expired: false,
972			ended: false,
973			stale_counted: Arc::default(),
974		}
975	}
976
977	/// Register for the first-frame timestamp while the group is still unstamped.
978	pub(crate) fn poll_timestamp(&self, waiter: &kio::Waiter) -> Poll<()> {
979		match self.state.poll(waiter, |state| {
980			if state.timestamp.is_some() || state.fin.is_some() || state.abort.is_some() {
981				Poll::Ready(())
982			} else {
983				Poll::Pending
984			}
985		}) {
986			Poll::Ready(_) => Poll::Ready(()),
987			Poll::Pending => Poll::Pending,
988		}
989	}
990
991	/// Block until the group is closed or aborted.
992	pub async fn closed(&self) -> Error {
993		kio::wait(|waiter| self.poll_closed(waiter)).await
994	}
995
996	/// Poll until the group is closed or aborted; ready with the cause.
997	pub fn poll_closed(&self, waiter: &kio::Waiter) -> Poll<Error> {
998		self.state.poll_closed(waiter).map(|()| self.abort_reason())
999	}
1000
1001	/// Block until there is at least one active consumer.
1002	pub async fn used(&self) -> Result<()> {
1003		self.state.used().await.map_err(|_| self.abort_reason())
1004	}
1005
1006	/// Block until there are no active consumers.
1007	pub async fn unused(&self) -> Result<()> {
1008		self.state.unused().await.map_err(|_| self.abort_reason())
1009	}
1010
1011	/// Poll for the group becoming unused (every consumer dropped).
1012	pub(crate) fn poll_unused(&self, waiter: &kio::Waiter) -> Poll<()> {
1013		self.state.poll_unused(waiter).map(|_| ())
1014	}
1015
1016	/// The recorded abort reason, or [`Error::Dropped`] if the group closed without one.
1017	fn abort_reason(&self) -> Error {
1018		self.state.read().abort.clone().unwrap_or(Error::Dropped)
1019	}
1020}
1021
1022impl Clone for Producer {
1023	fn clone(&self) -> Self {
1024		Self {
1025			info: self.info,
1026			state: self.state.clone(),
1027			track: self.track.clone(),
1028			cache: self.cache.clone(),
1029			stats: self.stats.clone(),
1030			alive: self.alive.clone(),
1031		}
1032	}
1033}
1034
1035/// A small inline batch of completed frames, drained from the shared group state
1036/// under one lock and then handed out without re-locking.
1037///
1038/// Each [`Consumer::read_frame`] otherwise takes the group mutex and allocates a
1039/// waker just to clone one `Bytes`; draining a batch amortizes both across `CAP`
1040/// frames. Storage is inline and uninitialized (no heap), so a consumer that never
1041/// reads whole frames, or drains through a higher-level buffer, pays nothing.
1042struct Prefetch {
1043	// Initialized, not-yet-taken frames are `frames[pos..len]`; the rest are uninitialized.
1044	frames: [MaybeUninit<Frame>; Self::CAP],
1045	pos: usize,
1046	len: usize,
1047}
1048
1049impl Prefetch {
1050	const CAP: usize = 8;
1051
1052	/// Take the next buffered frame, or `None` if the batch is drained.
1053	fn pop(&mut self) -> Option<Frame> {
1054		if self.pos == self.len {
1055			return None;
1056		}
1057		// SAFETY: `pos < len`, so this slot was written by `fill` and not yet taken.
1058		let frame = unsafe { self.frames[self.pos].assume_init_read() };
1059		self.pos += 1;
1060		Some(frame)
1061	}
1062
1063	/// Refill with up to `CAP` frames. Must be drained first (`pop` returned `None`).
1064	fn fill(&mut self, frames: impl Iterator<Item = Frame>) {
1065		debug_assert_eq!(self.pos, self.len, "fill on a non-empty batch would leak frames");
1066		self.pos = 0;
1067		self.len = 0;
1068		for frame in frames.take(Self::CAP) {
1069			self.frames[self.len].write(frame);
1070			self.len += 1;
1071		}
1072	}
1073
1074	/// `(frame count, total payload bytes)` of the buffered, not-yet-taken frames.
1075	/// Read once per fill to bump the egress payload counters for the whole batch.
1076	fn buffered(&self) -> (u64, u64) {
1077		let mut bytes = 0u64;
1078		for slot in &self.frames[self.pos..self.len] {
1079			// SAFETY: slots in `pos..len` are initialized (written by `fill`, not yet popped).
1080			bytes += unsafe { slot.assume_init_ref() }.payload.len() as u64;
1081		}
1082		((self.len - self.pos) as u64, bytes)
1083	}
1084}
1085
1086impl Default for Prefetch {
1087	fn default() -> Self {
1088		Self {
1089			frames: [const { MaybeUninit::uninit() }; Self::CAP],
1090			pos: 0,
1091			len: 0,
1092		}
1093	}
1094}
1095
1096impl Drop for Prefetch {
1097	fn drop(&mut self) {
1098		for slot in &mut self.frames[self.pos..self.len] {
1099			// SAFETY: slots in `pos..len` are initialized and were never taken.
1100			unsafe { slot.assume_init_drop() };
1101		}
1102	}
1103}
1104
1105/// Consume a group, frame-by-frame.
1106///
1107/// Usually a view of one [`Producer`], but a group served across a route change is
1108/// *spliced*: it reads each contributing route's copy in turn, joined at the frame the
1109/// takeover happened on, so the reader never sees the seam.
1110pub struct Consumer {
1111	inner: ConsumerKind,
1112
1113	// Immutable stream state.
1114	info: Info,
1115
1116	// The parent track's info, inherited from the producer. Its `timescale` lets the
1117	// wire publisher emit per-frame timestamps at the right scale for a fetched group.
1118	track: track::Info,
1119
1120	// Egress payload meter, set by a tagged track via [`Self::with_meter`]. Empty
1121	// (no-op) for an untagged group.
1122	stats: stats::Meter,
1123	// The meter that owns unread content discarded by expiry. Route-specific
1124	// cursors inside a spliced group inherit this without metering delivery twice.
1125	stale_stats: stats::Meter,
1126
1127	// Subscriber-specific drift policy. A group can become stale after the track
1128	// hands it out, while its reader is waiting for the first or next frame.
1129	expiry: Option<Arc<dyn Expiry>>,
1130	expired: bool,
1131	// Sticky: the budget gave up on a cursor that had already taken every frame, so
1132	// the group ends rather than fails. Recorded because `expired` alone would turn a
1133	// clean end into `Error::Old` on the next poll, and a caller is allowed to probe
1134	// again after the end.
1135	ended: bool,
1136	// Cloned cursors are parallel views of one handed-out delivery. Whichever
1137	// observes expiry first records its unread tail; the others must not repeat it.
1138	stale_counted: Arc<AtomicBool>,
1139}
1140
1141/// Subscriber-specific policy for expiring a group after it was handed out.
1142///
1143/// Unwind safe so the [`Consumer`] holding it is, and so the published
1144/// [`track::Fetching`] that holds a consumer stays unwind safe too.
1145pub(crate) trait Expiry: Send + Sync + std::panic::UnwindSafe + std::panic::RefUnwindSafe {
1146	/// Return whether the group is stale, registering `waiter` for anything that
1147	/// could change the answer while the group remains live.
1148	fn is_expired(&self, waiter: &kio::Waiter) -> bool;
1149}
1150
1151// `Plain` is the hot path and carries an inline frame prefetch, so boxing it to even the
1152// variants out would cost an allocation per group to save a pointer chase on the rare one.
1153#[expect(clippy::large_enum_variant)]
1154enum ConsumerKind {
1155	Plain(Plain),
1156	// Boxed: the spliced cursor set dwarfs the plain one, and splicing is the rare case.
1157	Spliced(Box<super::resume::Group>),
1158}
1159
1160/// The cursor state for a group backed by a single [`Producer`].
1161struct Plain {
1162	// Shared state with the producer.
1163	state: kio::Consumer<GroupState>,
1164
1165	// The index of the next frame to read.
1166	// NOTE: Cloned readers inherit this offset, but then run in parallel.
1167	index: usize,
1168
1169	// Exclusive cap on `index`, set by [`Consumer::set_frames`]. Reads end cleanly at it.
1170	end: Option<usize>,
1171
1172	// A batch of completed frames drained ahead under one lock (whole-frame reads only).
1173	prefetch: Prefetch,
1174
1175	// Record prefetched reads without entering the group's state on every frame.
1176	cache: Arc<cache::Track>,
1177	access: Arc<cache::Access>,
1178	refreshed: u64,
1179}
1180
1181impl Clone for Plain {
1182	fn clone(&self) -> Self {
1183		// A clone shares the channel and inherits `index`, but starts with an empty
1184		// prefetch: it re-reads its batch from the shared state, in parallel.
1185		Self {
1186			state: self.state.clone(),
1187			index: self.index,
1188			end: self.end,
1189			prefetch: Prefetch::default(),
1190			cache: self.cache.clone(),
1191			access: self.access.clone(),
1192			refreshed: self.refreshed,
1193		}
1194	}
1195}
1196
1197impl Clone for Consumer {
1198	fn clone(&self) -> Self {
1199		Self {
1200			inner: match &self.inner {
1201				ConsumerKind::Plain(plain) => ConsumerKind::Plain(plain.clone()),
1202				ConsumerKind::Spliced(spliced) => ConsumerKind::Spliced(Box::new((**spliced).clone())),
1203			},
1204			info: self.info,
1205			track: self.track.clone(),
1206			// Inherit the meter without re-counting the group: the original already
1207			// counted it when the track handed it out.
1208			stats: self.stats.clone(),
1209			stale_stats: self.stale_stats.clone(),
1210			expiry: self.expiry.clone(),
1211			expired: self.expired,
1212			ended: self.ended,
1213			stale_counted: self.stale_counted.clone(),
1214		}
1215	}
1216}
1217
1218impl std::ops::Deref for Consumer {
1219	type Target = Info;
1220
1221	fn deref(&self) -> &Self::Target {
1222		&self.info
1223	}
1224}
1225
1226impl Consumer {
1227	/// Snapshot the content this cursor would discard if its group were skipped.
1228	pub(crate) fn content(&self) -> stats::Content {
1229		match &self.inner {
1230			ConsumerKind::Plain(plain) => plain.state.read().content(),
1231			// Drift is evaluated before a segment copy is wrapped as a spliced group.
1232			// Keep the group count honest if a future caller reaches this fallback.
1233			ConsumerKind::Spliced(_) => stats::Content {
1234				groups: 1,
1235				..Default::default()
1236			},
1237		}
1238	}
1239
1240	/// Content not already attributed as delivered by this handed-out cursor.
1241	fn unread_content(&self) -> stats::Content {
1242		match &self.inner {
1243			ConsumerKind::Plain(plain) => plain.unread_content(),
1244			// Each route-specific plain cursor enforces expiry inside a spliced group.
1245			ConsumerKind::Spliced(_) => stats::Content::default(),
1246		}
1247	}
1248
1249	/// Rebuild this consumer as the head of a group assembled across route changes,
1250	/// keeping the group's identity and its track's properties. See [`super::resume`].
1251	pub(crate) fn into_spliced(self, mut spliced: super::resume::Group) -> Self {
1252		spliced.set_stale_meter(self.stale_stats.clone());
1253		Self {
1254			inner: ConsumerKind::Spliced(Box::new(spliced)),
1255			info: self.info,
1256			track: self.track,
1257			stats: self.stats,
1258			stale_stats: self.stale_stats,
1259			// Each segment keeps its own route-specific expiry policy. Applying the
1260			// head segment's policy to the assembled group would use the wrong edge
1261			// after a takeover.
1262			expiry: None,
1263			expired: false,
1264			ended: false,
1265			stale_counted: self.stale_counted,
1266		}
1267	}
1268
1269	/// Attach an egress payload meter, counting this as one delivered group.
1270	/// Called by a tagged track when it hands the consumer to a subscriber or fetch.
1271	pub(crate) fn with_meter(mut self, meter: stats::Meter) -> Self {
1272		meter.group();
1273		self.stats = meter.clone();
1274		self.set_stale_meter(meter);
1275		self
1276	}
1277
1278	/// Attach only the meter that owns content discarded by expiry.
1279	pub(crate) fn set_stale_meter(&mut self, meter: stats::Meter) {
1280		if let ConsumerKind::Spliced(spliced) = &mut self.inner {
1281			spliced.set_stale_meter(meter.clone());
1282		}
1283		self.stale_stats = meter;
1284	}
1285
1286	/// Keep applying this subscription's drift budget while the group is read.
1287	pub(crate) fn with_expiry(mut self, expiry: Arc<dyn Expiry>) -> Self {
1288		self.expiry = Some(expiry);
1289		self
1290	}
1291
1292	/// Check the parent subscription while a wire publisher drains detached payload.
1293	pub(crate) fn poll_expired(&mut self, waiter: &kio::Waiter) -> bool {
1294		self.poll_expired_while_pending(waiter, false)
1295	}
1296
1297	/// Apply the drift budget to a read that found nothing and is about to park.
1298	///
1299	/// A group with frames in hand is always drained to its end: the budget bounds a
1300	/// group that has *stalled* while the live edge moved on, not one whose reader is
1301	/// merely slower than the wire. Judging every read instead would truncate the tail
1302	/// of every group, since the arrival of the next group is exactly what makes the
1303	/// current one no longer newest.
1304	///
1305	/// Keeping it off the ready path also keeps it off the hot path: evaluating the
1306	/// policy walks the track's group cache under its lock, which is shared by every
1307	/// subscriber of that track.
1308	///
1309	/// `Some(false)` ends the group cleanly and `Some(true)` fails it with
1310	/// [`Error::Old`]; see [`Self::expired_truncates`].
1311	fn poll_expired_if_blocked(&mut self, waiter: &kio::Waiter) -> Option<bool> {
1312		if self.ended {
1313			return Some(false);
1314		}
1315		if !self.poll_expired(waiter) {
1316			return None;
1317		}
1318		let truncates = self.expired_truncates();
1319		self.ended = !truncates;
1320		Some(truncates)
1321	}
1322
1323	/// Whether giving up on this group now loses the reader anything.
1324	///
1325	/// A frame-level read only parks once the cursor has taken every frame the group
1326	/// holds, so expiring there costs nothing: the reader got everything that exists,
1327	/// and the group ends rather than fails. What it was still waiting for was the
1328	/// producer's FIN, and a group abandoned at its own end is indistinguishable from
1329	/// one that ended. A cursor that still holds unread content (a wire publisher with
1330	/// buffered frames, a half-read payload) is genuinely truncated and reports it.
1331	fn expired_truncates(&self) -> bool {
1332		let unread = self.unread_content();
1333		unread.frames > 0 || unread.bytes > 0
1334	}
1335
1336	/// Keep checking expiry while a wire publisher still owns buffered group data.
1337	pub(crate) fn poll_expired_while_pending(&mut self, waiter: &kio::Waiter, pending: bool) -> bool {
1338		if !self.expired
1339			&& (pending || self.expiry_pending())
1340			&& self.expiry.as_ref().is_some_and(|expiry| expiry.is_expired(waiter))
1341		{
1342			self.expired = true;
1343			if !self.stale_counted.swap(true, Ordering::Relaxed) {
1344				self.stale_stats.stale(self.unread_content());
1345			}
1346		}
1347		self.expired
1348	}
1349
1350	/// Whether expiry can still discard content or unblock a group that may grow.
1351	fn expiry_pending(&self) -> bool {
1352		match &self.inner {
1353			ConsumerKind::Plain(plain) => plain.expiry_pending(),
1354			// The route-specific plain cursors own expiry for a spliced group.
1355			ConsumerKind::Spliced(_) => false,
1356		}
1357	}
1358
1359	/// Whether this cursor failed because its subscription max age budget expired.
1360	pub(crate) fn latency_expired(&self) -> bool {
1361		self.expired
1362	}
1363
1364	/// Whether the group has been aborted (including pool eviction); the abort
1365	/// dropped the cached frames, so a held consumer has nothing left to read.
1366	///
1367	/// A spliced group spans several routes, so no single abort empties it; only a
1368	/// plain cursor can answer.
1369	pub(crate) fn is_aborted(&self) -> bool {
1370		match &self.inner {
1371			ConsumerKind::Plain(plain) => plain.state.read().abort.is_some(),
1372			ConsumerKind::Spliced(_) => false,
1373		}
1374	}
1375
1376	/// Mark the group as still being read, so a slow batch drain does not expire it.
1377	pub fn keep_alive(&self) {
1378		if let ConsumerKind::Plain(plain) = &self.inner {
1379			plain.state.read().charge.refresh();
1380		}
1381	}
1382
1383	/// Record a cache access from the consumer side: a parked group re-offered to
1384	/// its subscriber. Same stamp as [`Producer::cache_refresh`].
1385	pub(crate) fn cache_refresh(&self) {
1386		self.keep_alive();
1387	}
1388
1389	/// Park `waiter` until the group closes (finish, abort, or eviction). Spliced
1390	/// subscribers register on parked groups so an eviction wakes them; a group
1391	/// that already closed cleanly can never abort, so no waiter is needed.
1392	///
1393	/// A spliced group reads as closed without registering anything: no single abort
1394	/// empties it, so [`Self::is_aborted`] can never turn true and there is nothing
1395	/// a wakeup would change.
1396	pub(crate) fn poll_closed(&self, waiter: &kio::Waiter) -> Poll<()> {
1397		match &self.inner {
1398			ConsumerKind::Plain(plain) => plain.state.poll_closed(waiter),
1399			ConsumerKind::Spliced(_) => Poll::Ready(()),
1400		}
1401	}
1402
1403	/// The parent track's timescale.
1404	pub fn timescale(&self) -> Timescale {
1405		self.track.timescale
1406	}
1407
1408	/// The index of the next frame this consumer will return.
1409	///
1410	/// Starts at 0, or at the group's first available frame once [`Self::set_frames`] has
1411	/// clamped it, and advances by one per frame read.
1412	pub fn index(&self) -> u64 {
1413		match &self.inner {
1414			ConsumerKind::Plain(plain) => plain.index as u64,
1415			ConsumerKind::Spliced(spliced) => spliced.index(),
1416		}
1417	}
1418
1419	/// Limit subsequent reads to these frame indices without rewinding read progress.
1420	///
1421	/// `2..=5` includes frames 2 through 5; `2..5` excludes frame 5. An omitted
1422	/// start preserves read progress, and an omitted end removes the cap.
1423	/// Raising the cap makes unread cached frames available again.
1424	pub fn set_frames(&mut self, frames: impl RangeBounds<u64>) {
1425		let (start, end) = super::subscription::sequence_bounds(frames);
1426		self.start_at(start);
1427		self.end_at(end.map_or(Bound::Unbounded, Bound::Excluded));
1428	}
1429
1430	/// Skip ahead so the next frame returned is `index`, discarding anything buffered
1431	/// below it.
1432	///
1433	/// Clamped *up* to the group's first available frame: frames the group never held
1434	/// (see [`Producer::start_at`]) can't be returned, so asking for one just starts at
1435	/// the first that exists. Read [`Self::index`] back to learn where the cursor
1436	/// actually landed.
1437	/// Only moves forward; a lower `index` is ignored, since the frames behind the
1438	/// cursor may already have been handed out.
1439	pub(crate) fn start_at(&mut self, index: u64) {
1440		match &mut self.inner {
1441			ConsumerKind::Plain(plain) => plain.start_at(index),
1442			ConsumerKind::Spliced(spliced) => spliced.start_at(index),
1443		}
1444	}
1445
1446	/// Advance the read cursor to `index`, skipping every frame below it.
1447	///
1448	/// Unlike [`Self::set_frames`], this does not clamp past a requested frame the group
1449	/// never held. A [`Producer::start_at`] floor above `index` still surfaces as
1450	/// [`Error::Lagged`].
1451	pub fn skip_to(&mut self, index: u64) {
1452		match &mut self.inner {
1453			ConsumerKind::Plain(plain) => plain.skip_to(index),
1454			ConsumerKind::Spliced(spliced) => spliced.start_at(index),
1455		}
1456	}
1457
1458	/// Stop reading at `end`, or remove the cap with `..`.
1459	///
1460	/// `..=2` reads through frame 2, `..2` stops before it, and `..0` is the empty range:
1461	/// no frame is delivered. Reads past the cap end cleanly (`None`), as if the group
1462	/// finished there. The cap can move in either direction: raising it re-offers
1463	/// frames that are still cached.
1464	pub(crate) fn end_at(&mut self, end: impl Into<Cap>) {
1465		let end = end.into().exclusive();
1466		match &mut self.inner {
1467			ConsumerKind::Plain(plain) => {
1468				plain.end = end.map(|end| usize::try_from(end).unwrap_or(usize::MAX));
1469			}
1470			ConsumerKind::Spliced(spliced) => spliced.end_at(end),
1471		}
1472	}
1473
1474	/// The number of frames written so far (completed plus any in-flight), independent of
1475	/// how many this consumer has read. The final total once the group is finished.
1476	pub fn frame_count(&self) -> usize {
1477		match &self.inner {
1478			ConsumerKind::Plain(plain) => {
1479				let state = plain.state.read();
1480				state.fin.unwrap_or(state.next_index)
1481			}
1482			ConsumerKind::Spliced(spliced) => spliced.frame_count(),
1483		}
1484	}
1485
1486	/// Return a consumer for the next frame for chunked reading.
1487	pub async fn next_frame(&mut self) -> Result<Option<frame::Consumer>> {
1488		kio::wait(|waiter| self.poll_next_frame(waiter)).await
1489	}
1490
1491	/// Poll for the next frame, without blocking.
1492	///
1493	/// Returns None if the group is finished and the index is out of range, or the cursor
1494	/// passed the [`Self::set_frames`] cap.
1495	pub fn poll_next_frame(&mut self, waiter: &kio::Waiter) -> Poll<Result<Option<frame::Consumer>>> {
1496		if self.ended {
1497			return Poll::Ready(Ok(None));
1498		}
1499		if self.expired {
1500			return Poll::Ready(Err(Error::Old));
1501		}
1502		let stats = self.stats.clone();
1503		let expiry = self
1504			.expiry
1505			.as_ref()
1506			.map(|policy| frame::Expiry::new(policy.clone(), self.stale_stats.clone(), self.stale_counted.clone()));
1507		let res = match &mut self.inner {
1508			ConsumerKind::Plain(plain) => plain.poll_next_frame(waiter, &stats, expiry),
1509			ConsumerKind::Spliced(spliced) => {
1510				// The per-route copies underneath are untagged, so meter the spliced
1511				// stream here: it is the one the subscriber actually reads.
1512				let res = ready!(spliced.poll_next_frame(waiter))?;
1513				if res.is_some() {
1514					stats.frames(1);
1515				}
1516				Poll::Ready(Ok(res.map(|frame| frame.with_meter(stats))))
1517			}
1518		};
1519		match res.is_pending().then(|| self.poll_expired_if_blocked(waiter)).flatten() {
1520			Some(true) => Poll::Ready(Err(Error::Old)),
1521			Some(false) => Poll::Ready(Ok(None)),
1522			None => res,
1523		}
1524	}
1525
1526	/// Read the next frame (timestamp and payload) all at once, without blocking.
1527	pub fn poll_read_frame(&mut self, waiter: &kio::Waiter) -> Poll<Result<Option<frame::Frame>>> {
1528		if self.ended {
1529			return Poll::Ready(Ok(None));
1530		}
1531		if self.expired {
1532			return Poll::Ready(Err(Error::Old));
1533		}
1534		let stats = self.stats.clone();
1535		let res = match &mut self.inner {
1536			ConsumerKind::Plain(plain) => plain.poll_read_frame(waiter, &stats),
1537			ConsumerKind::Spliced(spliced) => {
1538				let res = ready!(spliced.poll_read_frame(waiter))?;
1539				if let Some(frame) = &res {
1540					stats.frames(1);
1541					stats.bytes(frame.payload.len() as u64);
1542				}
1543				Poll::Ready(Ok(res))
1544			}
1545		};
1546		match res.is_pending().then(|| self.poll_expired_if_blocked(waiter)).flatten() {
1547			Some(true) => Poll::Ready(Err(Error::Old)),
1548			Some(false) => Poll::Ready(Ok(None)),
1549			None => res,
1550		}
1551	}
1552
1553	/// Read the next frame (timestamp and payload) all at once.
1554	pub async fn read_frame(&mut self) -> Result<Option<frame::Frame>> {
1555		// A prefetched frame is already buffered, so the drift budget (which only judges
1556		// a read that would park) can never apply to it.
1557		if !self.expired
1558			&& let ConsumerKind::Plain(plain) = &mut self.inner
1559		{
1560			// Serve from the prefetched batch without building a future or allocating a waker.
1561			if !plain.capped()
1562				&& let Some(frame) = plain.prefetch.pop()
1563			{
1564				plain.refresh_if_stale();
1565				plain.index += 1;
1566				return Ok(Some(frame));
1567			}
1568		}
1569		kio::wait(|waiter| self.poll_read_frame(waiter)).await
1570	}
1571
1572	/// Fill `out` with every frame that is ready, up to its capacity, without blocking.
1573	///
1574	/// This is a short read: it returns as soon as anything is ready rather than
1575	/// waiting for `out` to fill. A zero count means the group ended when the buffer
1576	/// has non-zero capacity.
1577	pub fn poll_read_frames<const N: usize>(
1578		&mut self,
1579		waiter: &kio::Waiter,
1580		out: &mut frame::Buffer<N>,
1581	) -> Poll<Result<usize>> {
1582		out.clear();
1583		if out.capacity() == 0 {
1584			return Poll::Ready(Ok(0));
1585		}
1586
1587		while !out.is_full() {
1588			match self.poll_read_frame(waiter) {
1589				Poll::Ready(Ok(Some(frame))) => out.push(frame).expect("buffer capacity checked"),
1590				Poll::Ready(Ok(None)) => break,
1591				Poll::Ready(Err(err)) => {
1592					if out.is_empty() {
1593						return Poll::Ready(Err(err));
1594					}
1595					break;
1596				}
1597				Poll::Pending if !out.is_empty() => break,
1598				Poll::Pending => return Poll::Pending,
1599			}
1600		}
1601
1602		Poll::Ready(Ok(out.len()))
1603	}
1604
1605	/// Fill `out` with every frame that is ready, blocking until a frame arrives or
1606	/// the group ends. Returns the current batch, empty only at the end of the group.
1607	pub async fn read_frames<'a, const N: usize>(
1608		&mut self,
1609		out: &'a mut frame::Buffer<N>,
1610	) -> Result<&'a mut [frame::Frame]> {
1611		kio::wait(|waiter| self.poll_read_frames(waiter, out)).await?;
1612		Ok(out.filled_mut())
1613	}
1614
1615	/// Poll until the group terminates, returning this cursor's next frame index.
1616	pub fn poll_finished(&mut self, waiter: &kio::Waiter) -> Poll<Result<u64>> {
1617		if self.ended {
1618			return Poll::Ready(Ok(self.index()));
1619		}
1620		if self.expired {
1621			return Poll::Ready(Err(Error::Old));
1622		}
1623		let res = match &mut self.inner {
1624			ConsumerKind::Plain(plain) => {
1625				let index = plain.index;
1626				plain
1627					.poll(waiter, |state| state.poll_end(index))
1628					.map(|res| res.map(|()| index as u64))
1629			}
1630			ConsumerKind::Spliced(spliced) => spliced.poll_finished(waiter),
1631		};
1632		match res.is_pending().then(|| self.poll_expired_if_blocked(waiter)).flatten() {
1633			Some(true) => Poll::Ready(Err(Error::Old)),
1634			// The group ended where the cursor stands, so that is its frame count.
1635			Some(false) => Poll::Ready(Ok(self.index())),
1636			None => res,
1637		}
1638	}
1639
1640	/// Block until the group terminates, returning this cursor's next frame index.
1641	///
1642	/// This answers for the cursor, not the group: a reader that drained every frame gets the
1643	/// clean end even if the group was aborted afterwards to release its cache, while one that
1644	/// stopped short gets that abort. A prior [`Self::skip_to`] contributes to the index even
1645	/// though those frames were not read. Use [`Self::frame_count`] for the producer's total.
1646	pub async fn finished(&mut self) -> Result<u64> {
1647		kio::wait(|waiter| self.poll_finished(waiter)).await
1648	}
1649}
1650
1651impl Plain {
1652	/// Whether this cursor still has unread content or may receive another frame.
1653	fn expiry_pending(&self) -> bool {
1654		if self.capped() {
1655			return false;
1656		}
1657
1658		let state = self.state.read();
1659		state.abort.is_none() && state.fin.is_none_or(|fin| self.index < fin)
1660	}
1661
1662	/// Content this cursor has neither returned nor already counted in a prefetch batch.
1663	fn unread_content(&self) -> stats::Content {
1664		let prefetched = self.prefetch.buffered().0 as usize;
1665		let start = self.index.saturating_add(prefetched);
1666		let end = self.end.unwrap_or(usize::MAX);
1667		self.state.read().content_range(start, end)
1668	}
1669
1670	/// Record prefetched reads, updating the eviction rank once per sampled tick.
1671	fn refresh_if_stale(&mut self) {
1672		self.access.touch();
1673		let tick = self.cache.pool().now();
1674		if tick != self.refreshed {
1675			self.state.read().charge.refresh();
1676			self.refreshed = tick;
1677		}
1678	}
1679
1680	// A helper to automatically apply Dropped if the state is closed without an error.
1681	fn poll<F, R>(&self, waiter: &kio::Waiter, f: F) -> Poll<Result<R>>
1682	where
1683		F: Fn(&kio::Ref<'_, GroupState>) -> Poll<Result<R>>,
1684	{
1685		Poll::Ready(match ready!(self.state.poll(waiter, f)) {
1686			Ok(res) => res,
1687			// We try to clone abort just in case the function forgot to check for terminal state.
1688			Err(state) => Err(state.abort.clone().unwrap_or(Error::Dropped)),
1689		})
1690	}
1691
1692	/// Whether the cursor has passed the `end_at` cap.
1693	fn capped(&self) -> bool {
1694		self.end.is_some_and(|end| self.index >= end)
1695	}
1696
1697	fn start_at(&mut self, index: u64) {
1698		let index = usize::try_from(index).unwrap_or(usize::MAX);
1699		let index = index.max(self.state.read().offset);
1700		if index <= self.index {
1701			return;
1702		}
1703		self.index = index;
1704		// The batch was drained from below the new cursor, so it can't be reused.
1705		self.prefetch = Prefetch::default();
1706	}
1707
1708	fn skip_to(&mut self, index: u64) {
1709		let index = usize::try_from(index).unwrap_or(usize::MAX);
1710		if index <= self.index {
1711			return;
1712		}
1713		self.index = index;
1714		self.prefetch = Prefetch::default();
1715	}
1716
1717	fn poll_next_frame(
1718		&mut self,
1719		waiter: &kio::Waiter,
1720		stats: &stats::Meter,
1721		expiry: Option<frame::Expiry>,
1722	) -> Poll<Result<Option<frame::Consumer>>> {
1723		if self.capped() {
1724			return Poll::Ready(Ok(None));
1725		}
1726		let end = self.end.unwrap_or(usize::MAX);
1727
1728		// Hand out any frames a prior read_frame prefetched before touching the tail.
1729		// Their bytes were already counted at the batch fill, so the frame::Consumer
1730		// carries no meter.
1731		if let Some(frame) = self.prefetch.pop() {
1732			self.refresh_if_stale();
1733			self.index += 1;
1734			let tail = self.index.saturating_add(self.prefetch.buffered().0 as usize)..end;
1735			let info = frame::Info {
1736				size: frame.payload.len() as u64,
1737				timestamp: frame.timestamp,
1738			};
1739			let source = frame::Source::Complete(frame.payload);
1740			let frame = frame::Consumer::new(self.state.clone(), info, source);
1741			return Poll::Ready(Ok(Some(match expiry {
1742				Some(expiry) => frame.with_expiry(expiry.for_frame(tail, false)),
1743				None => frame,
1744			})));
1745		}
1746
1747		let index = self.index;
1748		let Some((info, source)) = ready!(self.poll(waiter, |state| state.poll_frame_source(index))?) else {
1749			return Poll::Ready(Ok(None));
1750		};
1751
1752		self.index += 1;
1753		// A direct read (not prefetched): count the frame here; the frame::Consumer
1754		// counts its bytes per chunk as they're read out.
1755		stats.frames(1);
1756		let frame = frame::Consumer::new(self.state.clone(), info, source).with_meter(stats.clone());
1757		Poll::Ready(Ok(Some(match expiry {
1758			Some(expiry) => frame.with_expiry(expiry.for_frame(self.index..end, true)),
1759			None => frame,
1760		})))
1761	}
1762
1763	fn poll_read_frame(&mut self, waiter: &kio::Waiter, stats: &stats::Meter) -> Poll<Result<Option<frame::Frame>>> {
1764		if self.capped() {
1765			return Poll::Ready(Ok(None));
1766		}
1767
1768		// Fast path: serve from the prefetched batch without locking or allocating a waker.
1769		if let Some(frame) = self.prefetch.pop() {
1770			self.refresh_if_stale();
1771			self.index += 1;
1772			return Poll::Ready(Ok(Some(frame)));
1773		}
1774
1775		// The batch is drained: refill it under a single lock, registering the waiter if
1776		// nothing is ready. Borrow the two fields disjointly so the closure can fill.
1777		let index = self.index;
1778		// Never buffer past the cap: `end_at` can be raised later, and those frames must
1779		// come from the shared state then, not from a batch drained under the old cap.
1780		let budget = self.end.map_or(usize::MAX, |end| end.saturating_sub(index));
1781		let prefetch = &mut self.prefetch;
1782		let res = self.state.poll(waiter, |state| {
1783			if index < state.offset {
1784				return Poll::Ready(Err(Error::Lagged));
1785			}
1786			// `local` can run past the buffered count when frames were cleared out from
1787			// under us (abort, unfinished drop); clamp so `range` never panics on an
1788			// out-of-bounds start. `fill` always resets the batch, so an empty range
1789			// leaves `len == 0` and the terminal checks below resolve abort/fin/pending.
1790			let local = (index - state.offset).min(state.frames.len());
1791			prefetch.fill(state.frames.range(local..).take(budget).cloned());
1792			if prefetch.len > 0 {
1793				// One stamp covers the whole batch: frames popped from the prefetch
1794				// don't re-stamp until the next refill, which `CAP` bounds.
1795				state.charge.refresh();
1796				return Poll::Ready(Ok(()));
1797			}
1798			// Nothing completed at `index`: an in-flight tail waits, otherwise resolve
1799			// the terminal state (whole-frame reads never stream the partial).
1800			state.poll_terminal(index)
1801		});
1802
1803		match ready!(res) {
1804			Ok(Ok(())) => {}
1805			Ok(Err(err)) => return Poll::Ready(Err(err)),
1806			Err(state) => return Poll::Ready(Err(state.abort.clone().unwrap_or(Error::Dropped))),
1807		}
1808
1809		// The refill already updated the eviction rank under the group lock.
1810		self.refreshed = self.cache.pool().now();
1811
1812		// A fresh batch was just filled (empty only on a clean end). Count the whole
1813		// batch once here, under no lock, so the drained pops that follow stay free.
1814		let (frames, bytes) = self.prefetch.buffered();
1815		stats.frames(frames);
1816		stats.bytes(bytes);
1817
1818		Poll::Ready(Ok(self.prefetch.pop().inspect(|_| {
1819			self.index += 1;
1820		})))
1821	}
1822}
1823
1824/// Options for a one-shot [`track::Consumer::fetch_group`] of a past group.
1825#[derive(Clone, Debug, Default)]
1826#[non_exhaustive]
1827pub struct Fetch {
1828	/// Delivery priority for the fetched group's stream. Defaults to 0.
1829	pub priority: u8,
1830
1831	/// Index of the first frame to fetch within the group. Defaults to 0, the whole group.
1832	///
1833	/// Use this to fill a hole left by a route change: the group's head is already
1834	/// cached locally and only the tail is missing.
1835	///
1836	/// There is no matching end: a fetch always runs to the end of the group, and a
1837	/// caller wanting less caps the returned consumer with [`Consumer::set_frames`]. Stopping
1838	/// the *fetch* short would put a group in the cache that is indistinguishable from a
1839	/// complete one, so a later fetch of the whole group would resolve from it and come
1840	/// up short.
1841	pub frame_start: u64,
1842}
1843
1844impl Fetch {
1845	/// Set the delivery priority, returning `self` for chaining.
1846	pub fn with_priority(mut self, priority: u8) -> Self {
1847		self.priority = priority;
1848		self
1849	}
1850
1851	/// Set the first frame to fetch, returning `self` for chaining.
1852	pub fn with_frame_start(mut self, frame_start: u64) -> Self {
1853		self.frame_start = frame_start;
1854		self
1855	}
1856}
1857
1858/// A consumer's request for a single past group, handed to a handler via
1859/// [`track::Dynamic::requested_group`].
1860///
1861/// The handler fulfills it by calling [`Self::accept`], which inserts the group
1862/// into the track cache (resolving every [`track::Consumer::fetch_group`] that joined the
1863/// attempt) and returns a [`Producer`] to fill. A relay typically opens a wire
1864/// FETCH and waits for the publisher to answer before accepting, so a group the
1865/// publisher lacks is rejected rather than accepted and then aborted. The request
1866/// carries its own producer handle, so it works the same whether or not the track
1867/// has been accepted yet.
1868pub struct Request {
1869	pub(crate) state: kio::Producer<track::TrackState>,
1870	pub(crate) fetch: kio::Shared<track::FetchState>,
1871	pub(crate) sequence: u64,
1872	pub(crate) priority: u8,
1873	pub(crate) frame_start: u64,
1874	pub(crate) result: kio::Producer<track::FetchOutcome>,
1875	pub(crate) done: bool,
1876}
1877
1878#[cfg(test)]
1879mod test {
1880	use super::*;
1881	use crate::model::test_tracing::count_drop_warnings;
1882	use bytes::Bytes;
1883	use futures::FutureExt;
1884
1885	/// [`FRAME_SLOTS`] is std's rounding, not ours, so measure it: a larger real value
1886	/// would undercharge every cached group without touching a line of this crate.
1887	#[test]
1888	fn one_frame_fits_the_charged_slots() {
1889		let mut frames: VecDeque<Frame> = VecDeque::new();
1890		frames.push_back(Frame {
1891			timestamp: Timestamp::ZERO,
1892			payload: Bytes::new(),
1893		});
1894		let capacity = frames.capacity();
1895		assert!(
1896			capacity <= FRAME_SLOTS,
1897			"a one-frame deque now allocates {capacity} slots"
1898		);
1899	}
1900
1901	#[test]
1902	fn basic_frame_reading() {
1903		let mut producer = Info { sequence: 0 }.produce();
1904		producer
1905			.write_frame(Timestamp::ZERO, Bytes::from_static(b"frame0"))
1906			.unwrap();
1907		producer
1908			.write_frame(Timestamp::ZERO, Bytes::from_static(b"frame1"))
1909			.unwrap();
1910		producer.finish().unwrap();
1911
1912		let mut consumer = producer.consume();
1913		let f0 = consumer.next_frame().now_or_never().unwrap().unwrap().unwrap();
1914		assert_eq!(f0.size, 6);
1915		let f1 = consumer.next_frame().now_or_never().unwrap().unwrap().unwrap();
1916		assert_eq!(f1.size, 6);
1917		let end = consumer.next_frame().now_or_never().unwrap().unwrap();
1918		assert!(end.is_none());
1919	}
1920
1921	#[test]
1922	fn read_frame_all_at_once() {
1923		let mut producer = Info { sequence: 0 }.produce();
1924		producer
1925			.write_frame(Timestamp::ZERO, Bytes::from_static(b"hello"))
1926			.unwrap();
1927		producer.finish().unwrap();
1928
1929		let mut consumer = producer.consume();
1930		let frame = consumer.read_frame().now_or_never().unwrap().unwrap().unwrap();
1931		assert_eq!(frame.payload, Bytes::from_static(b"hello"));
1932	}
1933
1934	#[test]
1935	fn read_frame_preserves_timestamp() {
1936		let mut producer = Info { sequence: 0 }.produce();
1937		let timestamp = Timestamp::from_micros(20_000).unwrap();
1938		producer.write_frame(timestamp, Bytes::from_static(b"hello")).unwrap();
1939		producer.finish().unwrap();
1940
1941		let mut consumer = producer.consume();
1942		let frame = consumer.read_frame().now_or_never().unwrap().unwrap().unwrap();
1943		assert_eq!(frame.timestamp.as_micros(), 20_000);
1944		assert_eq!(frame.payload, Bytes::from_static(b"hello"));
1945	}
1946
1947	#[test]
1948	fn chunked_frame_reads_whole() {
1949		let mut producer = Info { sequence: 0 }.produce();
1950		{
1951			let mut frame = producer
1952				.create_frame(frame::Info {
1953					size: 10,
1954					timestamp: Timestamp::ZERO,
1955				})
1956				.unwrap();
1957			frame.write(Bytes::from_static(b"hello")).unwrap();
1958			frame.write(Bytes::from_static(b"world")).unwrap();
1959			frame.finish().unwrap();
1960		}
1961		producer.finish().unwrap();
1962
1963		// Frame data is held in a single per-frame buffer; a whole-frame read returns
1964		// the full contents in one slice.
1965		let mut consumer = producer.consume();
1966		let frame = consumer.read_frame().now_or_never().unwrap().unwrap().unwrap();
1967		assert_eq!(frame.payload, Bytes::from_static(b"helloworld"));
1968	}
1969
1970	#[test]
1971	fn chunked_frame_streams_partial() {
1972		let mut producer = Info { sequence: 0 }.produce();
1973		let mut consumer = producer.consume();
1974
1975		let mut frame = producer
1976			.create_frame(frame::Info {
1977				size: 6,
1978				timestamp: Timestamp::ZERO,
1979			})
1980			.unwrap();
1981		frame.write(Bytes::from_static(b"foo")).unwrap();
1982
1983		// A consumer can stream the in-flight tail before it's finished.
1984		let mut f = consumer.next_frame().now_or_never().unwrap().unwrap().unwrap();
1985		let c1 = f.read_chunk().now_or_never().unwrap().unwrap();
1986		assert_eq!(c1, Some(Bytes::from_static(b"foo")));
1987		assert!(f.read_chunk().now_or_never().is_none());
1988
1989		frame.write(Bytes::from_static(b"bar")).unwrap();
1990		frame.finish().unwrap();
1991
1992		let c2 = f.read_chunk().now_or_never().unwrap().unwrap();
1993		assert_eq!(c2, Some(Bytes::from_static(b"bar")));
1994		let c3 = f.read_chunk().now_or_never().unwrap().unwrap();
1995		assert_eq!(c3, None);
1996	}
1997
1998	#[test]
1999	fn group_finish_returns_none() {
2000		let producer = Info { sequence: 0 }.produce();
2001		producer.finish().unwrap();
2002
2003		let mut consumer = producer.consume();
2004		let end = consumer.next_frame().now_or_never().unwrap().unwrap();
2005		assert!(end.is_none());
2006	}
2007
2008	#[test]
2009	fn abort_propagates() {
2010		let producer = Info { sequence: 0 }.produce();
2011		let mut consumer = producer.consume();
2012		producer.abort(crate::Error::Cancel).unwrap();
2013
2014		let result = consumer.next_frame().now_or_never().unwrap();
2015		assert!(matches!(result, Err(crate::Error::Cancel)));
2016	}
2017
2018	#[test]
2019	fn abort_unused_pairs_with_try_consume() {
2020		let producer = Info { sequence: 0 }.produce();
2021
2022		let consumer = producer.try_consume().expect("open");
2023		assert!(
2024			!producer.abort_unused(crate::Error::Cancel),
2025			"a reader declines the abort"
2026		);
2027		drop(consumer);
2028
2029		assert!(producer.abort_unused(crate::Error::Cancel));
2030		assert!(producer.try_consume().is_none(), "an aborted group mints no reader");
2031	}
2032
2033	#[test]
2034	fn abort_clears_cached_frames() {
2035		let mut producer = Info { sequence: 0 }.produce();
2036		producer
2037			.write_frame(Timestamp::ZERO, Bytes::from_static(b"data"))
2038			.unwrap();
2039
2040		// A stale consumer that never reads must not pin the cached frames.
2041		let _consumer = producer.consume();
2042		assert_eq!(producer.state.read().frames.len(), 1);
2043
2044		producer.clone().abort(crate::Error::Cancel).unwrap();
2045
2046		let state = producer.state.read();
2047		assert!(state.frames.is_empty(), "cached frames should be dropped on abort");
2048		assert_eq!(state.cache, 0);
2049	}
2050
2051	#[test]
2052	fn drop_unfinished_clears_cached_frames() {
2053		let producer = Info { sequence: 0 }.produce();
2054		let mut writer = producer.clone();
2055		writer
2056			.write_frame(Timestamp::ZERO, Bytes::from_static(b"data"))
2057			.unwrap();
2058
2059		// A stale consumer keeps the channel (and thus the cache) alive.
2060		let mut consumer = producer.consume();
2061		assert_eq!(producer.state.read().frames.len(), 1);
2062
2063		// Drop every producer without finishing: the cache is released.
2064		drop(writer);
2065		drop(producer);
2066
2067		let result = consumer.next_frame().now_or_never().unwrap();
2068		assert!(matches!(result, Err(crate::Error::Dropped)));
2069	}
2070
2071	#[test]
2072	fn drop_after_abort_does_not_warn() {
2073		let warns = count_drop_warnings("group::Producer dropped without finish", || {
2074			let producer = Info { sequence: 0 }.produce();
2075			let keep = producer.clone();
2076			let mut writer = producer.clone();
2077			writer
2078				.write_frame(Timestamp::ZERO, Bytes::from_static(b"data"))
2079				.unwrap();
2080			let _consumer = producer.consume();
2081			writer.abort(crate::Error::Cancel).unwrap();
2082			drop(keep);
2083		});
2084		assert_eq!(warns, 0, "abort-then-drop must not emit unfinished-producer WARN");
2085	}
2086
2087	#[test]
2088	fn drop_unfinished_warns() {
2089		let warns = count_drop_warnings("group::Producer dropped without finish", || {
2090			let producer = Info { sequence: 0 }.produce();
2091			let mut writer = producer.clone();
2092			writer
2093				.write_frame(Timestamp::ZERO, Bytes::from_static(b"data"))
2094				.unwrap();
2095			let _consumer = producer.consume();
2096			drop(writer);
2097			drop(producer);
2098		});
2099		assert!(warns >= 1, "unfinished drop must emit unfinished-producer WARN");
2100	}
2101
2102	#[test]
2103	fn drop_finished_keeps_cached_frames() {
2104		let mut producer = Info { sequence: 0 }.produce();
2105		producer
2106			.write_frame(Timestamp::ZERO, Bytes::from_static(b"data"))
2107			.unwrap();
2108		producer.finish().unwrap();
2109
2110		let mut consumer = producer.consume();
2111		drop(producer);
2112
2113		// A cleanly finished group keeps its cache so the consumer can still drain.
2114		let frame = consumer.read_frame().now_or_never().unwrap().unwrap().unwrap();
2115		assert_eq!(frame.payload, Bytes::from_static(b"data"));
2116	}
2117
2118	#[tokio::test]
2119	async fn pending_then_ready() {
2120		let mut producer = Info { sequence: 0 }.produce();
2121		let mut consumer = producer.consume();
2122
2123		// Consumer blocks because no frames yet.
2124		assert!(consumer.next_frame().now_or_never().is_none());
2125
2126		producer
2127			.write_frame(Timestamp::ZERO, Bytes::from_static(b"data"))
2128			.unwrap();
2129		producer.finish().unwrap();
2130
2131		let frame = consumer.next_frame().now_or_never().unwrap().unwrap().unwrap();
2132		assert_eq!(frame.size, 4);
2133	}
2134
2135	#[test]
2136	fn overflow_aborts_the_group() {
2137		let mut producer = Info { sequence: 0 }.produce();
2138		let mut consumer = producer.consume();
2139
2140		let big = Bytes::from(vec![0u8; MAX_CACHE_BYTES as usize]);
2141		producer.write_frame(Timestamp::ZERO, big.clone()).unwrap();
2142		assert!(matches!(
2143			producer.write_frame(Timestamp::ZERO, big),
2144			Err(Error::GroupTooLarge)
2145		));
2146
2147		{
2148			let state = producer.state.read();
2149			assert!(matches!(state.abort, Some(Error::GroupTooLarge)));
2150			assert!(state.frames.is_empty());
2151			assert_eq!(state.offset, 0);
2152		}
2153
2154		let result = consumer.next_frame().now_or_never().unwrap();
2155		assert!(matches!(result, Err(Error::GroupTooLarge)));
2156	}
2157
2158	#[test]
2159	fn no_overflow_under_budget() {
2160		let mut producer = Info { sequence: 0 }.produce();
2161		// 8192 one-byte frames is the largest legal group; they all stay cached.
2162		for _ in 0..MAX_GROUP_FRAMES {
2163			producer.write_frame(Timestamp::ZERO, Bytes::from_static(b"x")).unwrap();
2164		}
2165		producer.finish().unwrap();
2166
2167		let state = producer.state.read();
2168		assert_eq!(state.offset, 0);
2169		assert_eq!(state.frames.len(), MAX_GROUP_FRAMES);
2170		assert!(state.abort.is_none());
2171	}
2172
2173	#[test]
2174	fn writer_sees_group_too_large_on_the_8193rd_frame() {
2175		let mut producer = Info { sequence: 0 }.produce();
2176		for _ in 0..MAX_GROUP_FRAMES {
2177			producer.write_frame(Timestamp::ZERO, Bytes::from_static(b"x")).unwrap();
2178		}
2179		assert!(matches!(
2180			producer.write_frame(Timestamp::ZERO, Bytes::from_static(b"x")),
2181			Err(Error::GroupTooLarge)
2182		));
2183		assert!(matches!(producer.state.read().abort, Some(Error::GroupTooLarge)));
2184	}
2185
2186	#[test]
2187	fn clone_consumer_independent() {
2188		let mut producer = Info { sequence: 0 }.produce();
2189		producer.write_frame(Timestamp::ZERO, Bytes::from_static(b"a")).unwrap();
2190
2191		let mut c1 = producer.consume();
2192		// Read one frame from c1
2193		let _ = c1.next_frame().now_or_never().unwrap().unwrap().unwrap();
2194
2195		// Clone c1, inheriting its index (past first frame).
2196		let mut c2 = c1.clone();
2197
2198		producer.write_frame(Timestamp::ZERO, Bytes::from_static(b"b")).unwrap();
2199		producer.finish().unwrap();
2200
2201		// c2 should get the second frame (inherited index)
2202		let f = c2.next_frame().now_or_never().unwrap().unwrap().unwrap();
2203		assert_eq!(f.size, 1); // "b"
2204
2205		let end = c2.next_frame().now_or_never().unwrap().unwrap();
2206		assert!(end.is_none());
2207	}
2208
2209	fn prefetched_consumer(pool: &cache::Pool, max_age: std::time::Duration) -> (Producer, Consumer) {
2210		let cache = cache::Track::new(pool.clone(), kio::Weak::new());
2211		let track = track::Info::default().with_max_age(max_age);
2212		let mut producer = Producer::new(Info { sequence: 0 }, track, cache);
2213		producer.write_frame(Timestamp::ZERO, Bytes::from_static(b"a")).unwrap();
2214		producer.write_frame(Timestamp::ZERO, Bytes::from_static(b"b")).unwrap();
2215		producer.finish().unwrap();
2216
2217		let mut consumer = producer.consume();
2218		consumer.read_frame().now_or_never().unwrap().unwrap().unwrap();
2219		(producer, consumer)
2220	}
2221
2222	#[test]
2223	fn prefetch_refresh_honors_pool_expiry() {
2224		let config = cache::Config::default().with_expiry(std::time::Duration::from_secs(1));
2225		let pool = cache::Pool::new(config);
2226		let (producer, mut consumer) = prefetched_consumer(&pool, std::time::Duration::MAX);
2227		let before = producer.cache_accessed();
2228
2229		crate::model::clock::advance(std::time::Duration::from_millis(600));
2230		consumer.read_frame().now_or_never().unwrap().unwrap().unwrap();
2231
2232		assert!(producer.cache_accessed() > before, "the pool cadence is used");
2233	}
2234
2235	#[test]
2236	fn prefetch_refresh_honors_track_max_age() {
2237		let config = cache::Config::default().with_expiry(std::time::Duration::from_secs(30));
2238		let pool = cache::Pool::new(config);
2239		let (producer, mut consumer) = prefetched_consumer(&pool, std::time::Duration::from_secs(1));
2240		let before = producer.cache_accessed();
2241
2242		crate::model::clock::advance(std::time::Duration::from_millis(600));
2243		consumer.read_frame().now_or_never().unwrap().unwrap().unwrap();
2244
2245		assert!(producer.cache_accessed() > before, "the track cadence remains in force");
2246	}
2247
2248	/// Reading more than one prefetch batch drains every frame in order across the
2249	/// batch boundary (the refill starts exactly where the previous batch ended).
2250	#[test]
2251	fn read_frame_crosses_prefetch_batches() {
2252		let n = Prefetch::CAP * 3 + 5;
2253		let mut producer = Info { sequence: 0 }.produce();
2254		for i in 0..n {
2255			producer
2256				.write_frame(Timestamp::ZERO, Bytes::from(vec![i as u8; 4]))
2257				.unwrap();
2258		}
2259		producer.finish().unwrap();
2260
2261		let mut consumer = producer.consume();
2262		for i in 0..n {
2263			let frame = consumer.read_frame().now_or_never().unwrap().unwrap().unwrap();
2264			assert_eq!(frame.payload, Bytes::from(vec![i as u8; 4]));
2265		}
2266		assert!(consumer.read_frame().now_or_never().unwrap().unwrap().is_none());
2267	}
2268
2269	/// A finished group is still aborted once its frames are released to free memory (the
2270	/// track's max age window, or the cache pool). A reader that already drained every frame
2271	/// is missing nothing, so it must see the clean end of group rather than the abort.
2272	#[test]
2273	fn abort_after_finish_keeps_the_clean_end_for_a_drained_reader() {
2274		let mut producer = Info { sequence: 0 }.produce();
2275		producer
2276			.write_frame(Timestamp::ZERO, Bytes::from_static(b"hello"))
2277			.unwrap();
2278		producer.finish().unwrap();
2279
2280		let mut drained = producer.consume();
2281		let mut behind = producer.consume();
2282		let frame = drained.read_frame().now_or_never().unwrap().unwrap().unwrap();
2283		assert_eq!(frame.payload, Bytes::from_static(b"hello"));
2284
2285		producer.abort(Error::Old).unwrap();
2286
2287		// Drained everything before the abort: nothing is missing.
2288		assert!(drained.read_frame().now_or_never().unwrap().unwrap().is_none());
2289		assert!(drained.next_frame().now_or_never().unwrap().unwrap().is_none());
2290
2291		// Never read the frame, and its bytes are gone: a truncated stream, not a clean end.
2292		assert!(matches!(behind.read_frame().now_or_never().unwrap(), Err(Error::Old)));
2293	}
2294
2295	/// `finished` answers for the cursor: a drained reader gets the clean end even after the
2296	/// abort that released the cache, and one that stopped short gets that abort. The
2297	/// producer's total stays available on `frame_count`.
2298	#[test]
2299	fn finished_answers_for_the_cursor() {
2300		let mut producer = Info { sequence: 0 }.produce();
2301		producer.write_frame(Timestamp::ZERO, Bytes::from_static(b"a")).unwrap();
2302		producer.write_frame(Timestamp::ZERO, Bytes::from_static(b"b")).unwrap();
2303		producer.finish().unwrap();
2304
2305		let mut drained = producer.consume();
2306		let mut behind = producer.consume();
2307		while drained.read_frame().now_or_never().unwrap().unwrap().is_some() {}
2308		behind.read_frame().now_or_never().unwrap().unwrap().unwrap();
2309
2310		producer.abort(Error::Old).unwrap();
2311
2312		assert_eq!(drained.finished().now_or_never().unwrap().unwrap(), 2);
2313		assert!(matches!(behind.finished().now_or_never().unwrap(), Err(Error::Old)));
2314		assert_eq!(behind.frame_count(), 2);
2315	}
2316
2317	/// A cursor on a group aborted for overflowing its budget can never reach the end,
2318	/// so `finished` reports that abort instead of parking forever.
2319	#[test]
2320	fn finished_reports_a_group_too_large() {
2321		let mut producer = Info { sequence: 0 }.produce();
2322		let mut consumer = producer.consume();
2323
2324		let big = Bytes::from(vec![0u8; MAX_CACHE_BYTES as usize]);
2325		producer.write_frame(Timestamp::ZERO, big.clone()).unwrap();
2326		assert!(matches!(
2327			producer.write_frame(Timestamp::ZERO, big),
2328			Err(Error::GroupTooLarge)
2329		));
2330
2331		assert!(matches!(
2332			consumer.finished().now_or_never().unwrap(),
2333			Err(Error::GroupTooLarge)
2334		));
2335	}
2336
2337	/// `next_frame` drains frames a prior `read_frame` prefetched, preserving order.
2338	#[test]
2339	fn interleave_read_and_next_frame() {
2340		let mut producer = Info { sequence: 0 }.produce();
2341		for i in 0..5u8 {
2342			producer.write_frame(Timestamp::ZERO, Bytes::from(vec![i; 1])).unwrap();
2343		}
2344		producer.finish().unwrap();
2345
2346		let mut consumer = producer.consume();
2347		// The first whole-frame read prefetches all five frames into the batch.
2348		let f0 = consumer.read_frame().now_or_never().unwrap().unwrap().unwrap();
2349		assert_eq!(f0.payload, Bytes::from(vec![0u8; 1]));
2350
2351		// next_frame must continue from the batch, not skip ahead or repeat.
2352		for i in 1..5u8 {
2353			let mut f = consumer.next_frame().now_or_never().unwrap().unwrap().unwrap();
2354			let data = f.read_all().now_or_never().unwrap().unwrap();
2355			assert_eq!(data, Bytes::from(vec![i; 1]));
2356		}
2357		assert!(consumer.next_frame().now_or_never().unwrap().unwrap().is_none());
2358	}
2359
2360	/// A `read_frame` whose index sits past the buffered frames (cleared by an abort)
2361	/// must surface the error, not panic on an out-of-range `range(local..)`.
2362	#[test]
2363	fn read_frame_past_cleared_frames_does_not_panic() {
2364		let mut producer = Info { sequence: 0 }.produce();
2365		producer.write_frame(Timestamp::ZERO, Bytes::from_static(b"a")).unwrap();
2366		producer.write_frame(Timestamp::ZERO, Bytes::from_static(b"b")).unwrap();
2367
2368		let mut consumer = producer.consume();
2369		consumer.read_frame().now_or_never().unwrap().unwrap().unwrap();
2370		consumer.read_frame().now_or_never().unwrap().unwrap().unwrap();
2371
2372		// Abort clears the cached frames but leaves the consumer's index (2) past them, so the
2373		// refill's `local` (2) exceeds `frames.len()` (0).
2374		producer.abort(Error::Cancel).unwrap();
2375
2376		let result = consumer.read_frame().now_or_never().unwrap();
2377		assert!(matches!(result, Err(Error::Cancel)), "expected Cancel, got {result:?}");
2378	}
2379
2380	/// Dropping a consumer mid-batch must drop the buffered-but-untaken frames
2381	/// (exercises the `MaybeUninit` Drop path; run under miri to catch leaks/UB).
2382	#[test]
2383	fn drop_with_partial_batch() {
2384		let mut producer = Info { sequence: 0 }.produce();
2385		for _ in 0..Prefetch::CAP {
2386			producer.write_frame(Timestamp::ZERO, Bytes::from_static(b"x")).unwrap();
2387		}
2388		producer.finish().unwrap();
2389
2390		let mut consumer = producer.consume();
2391		// Take one frame so the batch is filled but only partially drained.
2392		let _ = consumer.read_frame().now_or_never().unwrap().unwrap().unwrap();
2393		drop(consumer);
2394	}
2395
2396	/// A parked chunk reader is woken by each chunk write. kio only notifies when
2397	/// a write guard was mutably accessed, so `frame_notify` must mark the guard
2398	/// modified; a guard dropped untouched wakes nobody and the reader would
2399	/// stall until the frame completed.
2400	#[tokio::test]
2401	async fn chunk_write_wakes_parked_reader() {
2402		let mut producer = Info { sequence: 0 }.produce();
2403		let mut consumer = producer.consume();
2404		let mut frame = producer
2405			.create_frame(frame::Info {
2406				size: 6,
2407				timestamp: Timestamp::ZERO,
2408			})
2409			.unwrap();
2410		let mut f = consumer.next_frame().await.unwrap().unwrap();
2411		let handle = tokio::spawn(async move { f.read_chunk().await });
2412		// Let the reader park on the empty partial before the chunk lands.
2413		tokio::time::sleep(std::time::Duration::from_millis(50)).await;
2414		frame.write(Bytes::from_static(b"foo")).unwrap();
2415		let chunk = tokio::time::timeout(std::time::Duration::from_secs(2), handle)
2416			.await
2417			.expect("parked chunk reader was never woken by the chunk write")
2418			.unwrap()
2419			.unwrap();
2420		assert_eq!(chunk, Some(Bytes::from_static(b"foo")));
2421	}
2422
2423	/// A frame whose timestamp is at a different scale is converted to the group's
2424	/// scale by `create_frame`.
2425	#[test]
2426	fn create_frame_converts_mismatched_scale() {
2427		use crate::{Timescale, Timestamp};
2428
2429		let mut producer = Producer::new(
2430			Info { sequence: 0 },
2431			track::Info::default().with_timescale(Timescale::MICRO),
2432			Default::default(),
2433		);
2434		let frame = frame::Info {
2435			size: 3,
2436			timestamp: Timestamp::from_millis(1).unwrap(), // 1ms -> 1000µs
2437		};
2438		let writer = producer.create_frame(frame).unwrap();
2439		assert_eq!(writer.timestamp.scale(), Timescale::MICRO);
2440		assert_eq!(writer.timestamp.value(), 1000);
2441	}
2442
2443	/// An explicit current timestamp is converted to the group's scale.
2444	#[tokio::test]
2445	async fn create_frame_converts_current_timestamp() {
2446		use crate::Timescale;
2447
2448		let mut producer = Producer::new(
2449			Info { sequence: 0 },
2450			track::Info::default().with_timescale(Timescale::MICRO),
2451			Default::default(),
2452		);
2453		let writer = producer
2454			.create_frame(frame::Info {
2455				size: 3,
2456				timestamp: Timestamp::now(),
2457			})
2458			.unwrap();
2459		assert_eq!(writer.timestamp.scale(), Timescale::MICRO);
2460		assert!(!writer.timestamp.is_zero(), "local clock should be non-zero");
2461	}
2462
2463	/// A group can start partway in, so a route can serve the tail of a group whose
2464	/// head came from somewhere else.
2465	#[test]
2466	fn start_at_starts_the_group_later() {
2467		let mut producer = Info { sequence: 0 }.produce();
2468		producer.start_at(3).unwrap();
2469		producer.write_frame(Timestamp::ZERO, Bytes::from_static(b"d")).unwrap();
2470		producer.finish().unwrap();
2471
2472		// The frame landed at index 3, so the group's length counts the missing head.
2473		assert_eq!(producer.frame_count(), 4);
2474
2475		let mut consumer = producer.consume();
2476		assert_eq!(consumer.frame_count(), 4);
2477
2478		// A reader positioned at the start is missing the head, and `finished` answers
2479		// for that cursor.
2480		assert!(matches!(
2481			consumer.finished().now_or_never().unwrap(),
2482			Err(Error::Lagged)
2483		));
2484		assert!(matches!(
2485			consumer.read_frame().now_or_never().unwrap(),
2486			Err(Error::Lagged)
2487		));
2488	}
2489
2490	/// Seeking to the group's first available frame is how a spliced reader picks up
2491	/// the tail; a lower index clamps up rather than failing.
2492	#[test]
2493	fn start_at_clamps_up_to_the_first_frame() {
2494		let mut producer = Info { sequence: 0 }.produce();
2495		producer.start_at(3).unwrap();
2496		producer.write_frame(Timestamp::ZERO, Bytes::from_static(b"d")).unwrap();
2497		producer.finish().unwrap();
2498
2499		let mut consumer = producer.consume();
2500		consumer.start_at(1);
2501		assert_eq!(consumer.index(), 3, "clamped up to the first frame that exists");
2502		assert_eq!(
2503			consumer.read_frame().now_or_never().unwrap().unwrap().unwrap().payload,
2504			Bytes::from_static(b"d")
2505		);
2506	}
2507
2508	/// `end_at` ends the read cleanly at the cap, and raising it re-offers the frames
2509	/// still cached behind it.
2510	#[test]
2511	fn end_at_caps_and_reopens() {
2512		let mut producer = Info { sequence: 0 }.produce();
2513		for i in 0..4u8 {
2514			producer.write_frame(Timestamp::ZERO, Bytes::from(vec![i])).unwrap();
2515		}
2516		producer.finish().unwrap();
2517
2518		let mut consumer = producer.consume();
2519		consumer.set_frames(..2);
2520		assert_eq!(
2521			consumer.read_frame().now_or_never().unwrap().unwrap().unwrap().payload[0],
2522			0
2523		);
2524		assert_eq!(
2525			consumer.read_frame().now_or_never().unwrap().unwrap().unwrap().payload[0],
2526			1
2527		);
2528		assert!(
2529			consumer.read_frame().now_or_never().unwrap().unwrap().is_none(),
2530			"capped reads end cleanly"
2531		);
2532
2533		consumer.set_frames(..);
2534		assert_eq!(
2535			consumer.read_frame().now_or_never().unwrap().unwrap().unwrap().payload[0],
2536			2
2537		);
2538	}
2539
2540	#[test]
2541	fn frame_ranges_preserve_progress_and_make_inclusion_explicit() {
2542		let mut producer = Info { sequence: 0 }.produce();
2543		for i in 0..4u8 {
2544			producer.write_frame(Timestamp::ZERO, Bytes::from(vec![i])).unwrap();
2545		}
2546		producer.finish().unwrap();
2547		let mut consumer = producer.consume();
2548		consumer.set_frames(1..=1);
2549		assert_eq!(
2550			consumer.read_frame().now_or_never().unwrap().unwrap().unwrap().payload[0],
2551			1
2552		);
2553		assert!(consumer.read_frame().now_or_never().unwrap().unwrap().is_none());
2554		consumer.set_frames(..3);
2555		assert_eq!(
2556			consumer.read_frame().now_or_never().unwrap().unwrap().unwrap().payload[0],
2557			2
2558		);
2559		assert!(consumer.read_frame().now_or_never().unwrap().unwrap().is_none());
2560		consumer.set_frames(0..=3);
2561		assert_eq!(
2562			consumer.read_frame().now_or_never().unwrap().unwrap().unwrap().payload[0],
2563			3
2564		);
2565	}
2566
2567	/// An exclusive cap at 0 is the empty range: no frame is delivered, and raising
2568	/// it re-offers the held frames.
2569	#[test]
2570	fn end_at_zero_is_empty() {
2571		let mut producer = Info { sequence: 0 }.produce();
2572		producer.write_frame(Timestamp::ZERO, Bytes::from_static(b"x")).unwrap();
2573		producer.finish().unwrap();
2574
2575		let mut consumer = producer.consume();
2576		consumer.set_frames(..0);
2577		assert!(
2578			consumer.read_frame().now_or_never().unwrap().unwrap().is_none(),
2579			"empty cap delivers nothing"
2580		);
2581
2582		consumer.set_frames(..1);
2583		assert_eq!(
2584			consumer.read_frame().now_or_never().unwrap().unwrap().unwrap().payload,
2585			Bytes::from_static(b"x")
2586		);
2587	}
2588
2589	/// Where the group begins is part of its shape, so it can't move once frames exist.
2590	#[test]
2591	fn start_at_rejected_after_a_frame() {
2592		let mut producer = Info { sequence: 0 }.produce();
2593		// Re-declaring before the first frame is fine; the shape isn't committed yet.
2594		producer.start_at(2).unwrap();
2595		producer.start_at(3).unwrap();
2596
2597		producer.write_frame(Timestamp::ZERO, Bytes::from_static(b"a")).unwrap();
2598		assert!(matches!(producer.start_at(4), Err(Error::Closed)));
2599		assert_eq!(producer.frame_count(), 4, "the frame landed at index 3");
2600
2601		// Finishing likewise settles the shape.
2602		let mut producer = Info { sequence: 1 }.produce();
2603		producer.finish().unwrap();
2604		assert!(matches!(producer.start_at(1), Err(Error::Closed)));
2605	}
2606
2607	/// The start must leave room for at least one frame index.
2608	#[test]
2609	fn start_at_rejects_the_largest_index() {
2610		let mut producer = Info { sequence: 0 }.produce();
2611		assert!(matches!(
2612			producer.start_at(usize::MAX as u64),
2613			Err(Error::BoundsExceeded(_))
2614		));
2615	}
2616
2617	/// The per-frame size cap (the group byte budget) is enforced before allocating.
2618	#[test]
2619	fn create_frame_rejects_oversized() {
2620		let mut producer = Info { sequence: 0 }.produce();
2621		let result = producer.create_frame(frame::Info {
2622			size: MAX_CACHE_BYTES + 1,
2623			timestamp: Timestamp::ZERO,
2624		});
2625		assert!(matches!(result, Err(Error::FrameTooLarge)));
2626	}
2627
2628	fn sized(size: u64) -> frame::Info {
2629		frame::Info {
2630			size,
2631			timestamp: Timestamp::ZERO,
2632		}
2633	}
2634
2635	/// A declared size costs the peer nothing, so the pool is charged as the bytes
2636	/// arrive. Charging the declaration would let a peer evict the cache without
2637	/// sending anything.
2638	#[test]
2639	fn frame_is_charged_as_written() {
2640		let pool = cache::Pool::unbounded();
2641		let cache = cache::Track::new(pool.clone(), kio::Weak::new());
2642		let mut producer = Producer::new(Info { sequence: 0 }, track::Info::default(), cache);
2643		let before = pool.used();
2644
2645		let mut frame = producer
2646			.create_frame_owned(sized(MAX_CACHE_BYTES), &frame::Budget::default())
2647			.unwrap();
2648		assert_eq!(pool.used(), before, "the declared size was charged");
2649
2650		frame.write(Bytes::from(vec![0u8; 100])).unwrap();
2651		assert_eq!(pool.used(), before, "charged before the write was published");
2652		frame.notify();
2653		assert_eq!(pool.used(), before + 100);
2654		assert_eq!(producer.state.read().cache, 100);
2655		// A reader skipping the group still misses the whole declared frame.
2656		assert_eq!(producer.state.read().content().bytes, MAX_CACHE_BYTES);
2657
2658		// Aborting releases what was charged.
2659		frame.abort(Error::Cancel).unwrap();
2660		assert_eq!(producer.state.read().cache, 0);
2661		assert!(pool.used() <= before);
2662	}
2663
2664	/// Completing a frame charges whatever it wrote since the last notify.
2665	#[test]
2666	fn frame_commit_charges_the_tail() {
2667		let pool = cache::Pool::unbounded();
2668		let cache = cache::Track::new(pool.clone(), kio::Weak::new());
2669		let mut producer = Producer::new(Info { sequence: 0 }, track::Info::default(), cache);
2670		let before = pool.used();
2671
2672		let mut frame = producer
2673			.create_frame_owned(sized(300), &frame::Budget::default())
2674			.unwrap();
2675		frame.write(Bytes::from(vec![0u8; 100])).unwrap();
2676		frame.notify();
2677		frame.write(Bytes::from(vec![0u8; 200])).unwrap();
2678		frame.finish().unwrap();
2679		assert_eq!(pool.used(), before + 300);
2680		assert_eq!(producer.state.read().cache, 300);
2681	}
2682
2683	/// Frames within the session's budget allocate their declared size up front; past
2684	/// it, a frame's buffer holds only what has arrived. A frame hands its share back
2685	/// once it ends.
2686	#[test]
2687	fn frame_budget_bounds_upfront_allocation() {
2688		let budget = frame::Budget::new(1024);
2689		let mut a = Info { sequence: 0 }.produce();
2690		let mut b = Info { sequence: 1 }.produce();
2691
2692		let mut first = a.create_frame_owned(sized(1024), &budget).unwrap();
2693		first.write(Bytes::from_static(b"x")).unwrap();
2694		assert_eq!(first.allocated(), 1024, "a frame within budget is allocated up front");
2695
2696		let mut second = b.create_frame_owned(sized(MAX_CACHE_BYTES), &budget).unwrap();
2697		second.write(Bytes::from(vec![0u8; 100])).unwrap();
2698		assert_eq!(
2699			second.allocated(),
2700			100,
2701			"a frame past budget allocated its declared size"
2702		);
2703		second.abort(Error::Cancel).unwrap();
2704
2705		first.write(Bytes::from(vec![0u8; 1023])).unwrap();
2706		first.finish().unwrap();
2707		a.finish().unwrap();
2708
2709		let mut a = Info { sequence: 2 }.produce();
2710		let mut third = a.create_frame_owned(sized(1024), &budget).unwrap();
2711		third.write(Bytes::from_static(b"x")).unwrap();
2712		assert_eq!(third.allocated(), 1024, "the finished frame did not return its share");
2713	}
2714}