1use std::collections::VecDeque;
4use std::fmt;
5use std::fs::File;
6use std::sync::Arc;
7use std::sync::atomic::{AtomicU64, Ordering};
8use std::time::{Duration, Instant};
9
10use crate::buf::{Buffer, BufferPool};
11use crate::completer::Completer;
12use crate::error::{CloseReason, LimitKind, NetError, RefusalCause, TimeoutKind};
13use crate::gate::KillHandle;
14use crate::id::WorkerId;
15use crate::info::{ConnectionInfo, GateTag};
16use crate::limits::SizeClass;
17use crate::link::Link;
18
19pub trait Session: Send + 'static {
46 fn on_open(&mut self, cx: &mut SessionContext<'_>) -> Step {
48 let _ = cx;
49 Step::Read
50 }
51
52 fn on_data(&mut self, cx: &mut SessionContext<'_>, input: &mut Input) -> Step;
54
55 fn on_writable(&mut self, cx: &mut SessionContext<'_>) -> Step {
57 let _ = cx;
58 Step::Read
59 }
60
61 fn on_wake(&mut self, cx: &mut SessionContext<'_>, token: u32, value: u64) -> Step {
63 let _ = (cx, token, value);
64 Step::Abort
65 }
66
67 fn on_timeout(&mut self, cx: &mut SessionContext<'_>, which: TimeoutKind) -> Step {
70 let _ = (cx, which);
71 Step::Close
72 }
73
74 fn on_peer_closed(&mut self, cx: &mut SessionContext<'_>) -> Step {
76 let _ = cx;
77 Step::Close
78 }
79
80 fn on_denied(&mut self, code: u16, output: &mut Output) {
83 let _ = (code, output);
84 }
85
86 fn on_close(&mut self, info: &ConnectionInfo, reason: CloseReason) {
88 let _ = (info, reason);
89 }
90}
91
92pub trait SessionFactory: Send + Sync + 'static {
123 type Session: Session;
125
126 fn open(&self, info: &ConnectionInfo) -> Result<Self::Session, RefusalCause>;
132}
133
134impl<F, S> SessionFactory for F
135where
136 F: Fn(&ConnectionInfo) -> S + Send + Sync + 'static,
137 S: Session,
138{
139 type Session = S;
140 fn open(&self, info: &ConnectionInfo) -> Result<S, RefusalCause> {
141 Ok(self(info))
142 }
143}
144
145#[non_exhaustive]
151#[derive(Clone, Copy, Debug, PartialEq, Eq)]
152pub enum Step {
153 Read,
156 Write,
158 Park {
160 until: Option<Instant>,
162 },
163 Close,
165 Abort,
167}
168
169#[derive(Clone, Copy, Debug, PartialEq, Eq, Default)]
175pub enum Phase {
176 #[default]
178 Idle,
179 Receiving,
181 Sending,
183}
184
185#[non_exhaustive]
191#[derive(Clone, Copy, Debug, PartialEq, Eq)]
192pub enum SwapError {
193 ClassMismatch,
195 NotEmpty,
197 Busy,
199}
200
201impl fmt::Display for SwapError {
202 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
203 f.write_str(match self {
204 SwapError::ClassMismatch => "the buffers are of different size classes",
205 SwapError::NotEmpty => "the incoming buffer is not empty",
206 SwapError::Busy => "a driver operation holds the buffer",
207 })
208 }
209}
210
211impl std::error::Error for SwapError {}
212
213#[derive(Debug)]
222pub struct Input {
223 pub(crate) buf: Buffer,
224 pub(crate) start: usize,
225 pub(crate) changed: bool,
226 lent: bool,
229}
230
231impl Input {
232 pub(crate) fn new(buf: Buffer) -> Input {
233 Input {
234 buf,
235 lent: false,
236 start: 0,
237 changed: false,
238 }
239 }
240
241 #[must_use]
243 pub fn from_bytes(bytes: &[u8]) -> Input {
244 let class = (bytes.len() as u32)
245 .max(1024)
246 .next_power_of_two()
247 .max(SizeClass::SMALL.bytes());
248 let class = SizeClass::new(class.min(1 << 20)).unwrap_or(SizeClass::LARGE);
249 let mut pool = BufferPool::standalone(u64::from(class.bytes()));
250 let mut input = Input::new(match pool.take(class) {
251 Some(b) => b,
252 None => return Input::new(Buffer::detached(class)),
253 });
254 let n = bytes.len().min(input.buf.capacity());
255 input.buf.spare()[..n].copy_from_slice(&bytes[..n]);
256 input.buf.set_len(n);
257 input
258 }
259
260 #[must_use]
262 pub fn bytes(&self) -> &[u8] {
263 self.buf.as_slice().get(self.start..).unwrap_or(&[])
264 }
265
266 pub fn consume(&mut self, n: usize) {
268 let n = n.min(self.buf.len() - self.start);
269 self.start += n;
270 if n > 0 {
271 self.changed = true;
272 }
273 if self.start == self.buf.len() {
274 self.buf.clear();
275 self.start = 0;
276 }
277 }
278
279 #[must_use]
281 pub fn is_full(&self) -> bool {
282 !self.lent && self.buf.len() == self.buf.capacity() && self.start == 0
283 }
284
285 #[must_use]
287 pub fn class(&self) -> SizeClass {
288 self.buf.class()
289 }
290
291 pub fn swap(&mut self, other: &mut Buffer) -> Result<(), SwapError> {
309 if other.class() != self.buf.class() {
310 return Err(SwapError::ClassMismatch);
311 }
312 if !other.is_empty() {
313 return Err(SwapError::NotEmpty);
314 }
315 if self.start > 0 {
316 self.buf.compact(self.start);
317 self.start = 0;
318 }
319 std::mem::swap(&mut self.buf, other);
320 self.changed = true;
321 Ok(())
322 }
323
324 pub(crate) fn spare_for_read(&mut self) -> &mut [u8] {
326 let free = self.buf.capacity() - self.buf.len();
327 if self.start > 0 && free < self.buf.capacity() / 4 {
328 self.buf.compact(self.start);
329 self.start = 0;
330 }
331 self.buf.spare()
332 }
333
334 #[cfg(feature = "reactor")]
337 pub(crate) fn lend(&mut self) -> Buffer {
338 let _ = self.spare_for_read();
339 self.lent = true;
340 std::mem::replace(&mut self.buf, Buffer::placeholder())
341 }
342
343 #[cfg(feature = "reactor")]
345 pub(crate) fn give_back(&mut self, buf: Buffer) {
346 self.buf = buf;
347 self.lent = false;
348 }
349
350 #[cfg(feature = "reactor")]
352 pub(crate) fn lent(&self) -> bool {
353 self.lent
354 }
355
356 pub(crate) fn filled(&mut self, n: usize) {
357 let len = self.buf.len() + n;
358 self.buf.set_len(len);
359 self.changed = true;
360 }
361}
362
363#[derive(Clone, Copy, Debug, PartialEq, Eq)]
369pub struct OutputFull;
370
371impl fmt::Display for OutputFull {
372 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
373 f.write_str("the connection's output budget is full")
374 }
375}
376
377impl std::error::Error for OutputFull {}
378
379#[derive(Debug)]
380pub(crate) enum Chunk {
381 Pooled(Buffer, usize),
382 Static(&'static [u8], usize),
383 File {
385 file: Arc<File>,
386 offset: u64,
387 left: u64,
388 },
389}
390
391impl Chunk {
392 pub(crate) fn remaining(&self) -> &[u8] {
394 match self {
395 Chunk::Pooled(b, off) => &b.as_slice()[*off..],
396 Chunk::Static(s, off) => &s[*off..],
397 Chunk::File { .. } => &[],
398 }
399 }
400
401 pub(crate) fn len(&self) -> usize {
403 match self {
404 Chunk::File { left, .. } => usize::try_from(*left).unwrap_or(usize::MAX),
405 other => other.remaining().len(),
406 }
407 }
408
409 pub(crate) fn is_file(&self) -> bool {
410 matches!(self, Chunk::File { .. })
411 }
412
413 pub(crate) fn advance(&mut self, n: usize) {
415 match self {
416 Chunk::Pooled(_, off) | Chunk::Static(_, off) => *off += n,
417 Chunk::File { offset, left, .. } => {
418 *offset += n as u64;
419 *left -= n as u64;
420 }
421 }
422 }
423}
424
425#[derive(Debug)]
434pub struct Output {
435 pub(crate) chunks: VecDeque<Chunk>,
436 free: Vec<Buffer>,
437 budget: Arc<AtomicU64>,
438 limit: u32,
439 used: u32,
440 wipe: bool,
441 refused: bool,
443 pub(crate) response_open: bool,
444 pub(crate) response_tag: Option<GateTag>,
445 pub(crate) bytes_since_begin: u64,
446}
447
448const OUTPUT_CLASS: SizeClass = SizeClass::MEDIUM;
449const MAX_CHUNKS: usize = 64;
450const FREE_MAX: usize = 4;
452
453impl Output {
454 pub(crate) fn new(budget: Arc<AtomicU64>, limit: u32, wipe: bool) -> Output {
455 Output {
456 chunks: VecDeque::with_capacity(MAX_CHUNKS),
457 free: Vec::new(),
458 budget,
459 limit,
460 used: 0,
461 wipe,
462 refused: false,
463 response_open: false,
464 response_tag: None,
465 bytes_since_begin: 0,
466 }
467 }
468
469 #[must_use]
471 pub fn standalone(limit: u32) -> Output {
472 Output::new(Arc::new(AtomicU64::new(u64::from(limit))), limit, true)
473 }
474
475 fn take_buffer(&mut self) -> Option<Buffer> {
476 if let Some(b) = self.free.pop() {
477 return Some(b);
478 }
479 let bytes = u64::from(OUTPUT_CLASS.bytes());
480 let mut left = self.budget.load(Ordering::Relaxed);
481 loop {
482 if left < bytes {
483 self.refused = true;
484 return None;
485 }
486 match self.budget.compare_exchange_weak(
487 left,
488 left - bytes,
489 Ordering::AcqRel,
490 Ordering::Relaxed,
491 ) {
492 Ok(_) => return Some(Buffer::detached(OUTPUT_CLASS)),
493 Err(actual) => left = actual,
494 }
495 }
496 }
497
498 pub(crate) fn refill(&mut self, pool: &mut BufferPool) {
507 while self.free.len() < FREE_MAX {
508 match pool.take_cached(OUTPUT_CLASS) {
509 Some(b) => self.free.push(b),
510 None => break,
511 }
512 }
513 if self.free.is_empty() {
514 pool.make_room(OUTPUT_CLASS);
515 }
516 }
517
518 pub(crate) fn take_refused(&mut self) -> bool {
521 std::mem::take(&mut self.refused)
522 }
523
524 fn room(&self) -> usize {
525 let tail = match self.chunks.back() {
526 Some(Chunk::Pooled(b, _)) => b.capacity() - b.len(),
527 _ => 0,
528 };
529 let more = (self.limit.saturating_sub(self.used) / OUTPUT_CLASS.bytes()) as usize
530 * OUTPUT_CLASS.bytes() as usize;
531 tail + more
532 }
533
534 pub fn push_partial(&mut self, mut bytes: &[u8]) -> usize {
536 let mut taken = 0;
537 while !bytes.is_empty() {
538 if let Some(Chunk::Pooled(b, _)) = self.chunks.back_mut() {
539 let spare = b.spare();
540 if !spare.is_empty() {
541 let n = spare.len().min(bytes.len());
542 spare[..n].copy_from_slice(&bytes[..n]);
543 let len = b.len() + n;
544 b.set_len(len);
545 bytes = &bytes[n..];
546 taken += n;
547 continue;
548 }
549 }
550 if self.chunks.len() >= MAX_CHUNKS
551 || self.used.saturating_add(OUTPUT_CLASS.bytes()) > self.limit
552 {
553 break;
554 }
555 match self.take_buffer() {
556 Some(b) => {
557 self.used += OUTPUT_CLASS.bytes();
558 self.chunks.push_back(Chunk::Pooled(b, 0));
559 }
560 None => break,
561 }
562 }
563 taken
564 }
565
566 pub fn push(&mut self, bytes: &[u8]) -> Result<(), OutputFull> {
572 if bytes.len() > self.room() {
573 return Err(OutputFull);
574 }
575 let n = self.push_partial(bytes);
576 if n == bytes.len() {
577 Ok(())
578 } else {
579 Err(OutputFull)
580 }
581 }
582
583 pub fn push_buffer(&mut self, buffer: Buffer) -> Result<(), OutputFull> {
589 let size = u32::try_from(buffer.capacity()).unwrap_or(u32::MAX);
590 if self.chunks.len() >= MAX_CHUNKS || self.used.saturating_add(size) > self.limit {
591 return Err(OutputFull);
592 }
593 self.used += size;
594 self.chunks.push_back(Chunk::Pooled(buffer, 0));
595 Ok(())
596 }
597
598 pub fn push_static(&mut self, bytes: &'static [u8]) -> Result<(), OutputFull> {
604 if self.chunks.len() >= MAX_CHUNKS {
605 return Err(OutputFull);
606 }
607 if !bytes.is_empty() {
608 self.chunks.push_back(Chunk::Static(bytes, 0));
609 }
610 Ok(())
611 }
612
613 pub fn push_file(&mut self, file: Arc<File>, offset: u64, len: u64) -> Result<(), OutputFull> {
623 if self.chunks.len() >= MAX_CHUNKS {
624 return Err(OutputFull);
625 }
626 if len > 0 {
627 self.chunks.push_back(Chunk::File {
628 file,
629 offset,
630 left: len,
631 });
632 }
633 Ok(())
634 }
635
636 pub(crate) fn front_file(&self) -> Option<(Arc<File>, u64, u64)> {
638 match self.chunks.front() {
639 Some(Chunk::File { file, offset, left }) => Some((file.clone(), *offset, *left)),
640 _ => None,
641 }
642 }
643
644 #[cfg(feature = "reactor")]
647 pub(crate) fn pull_file(&mut self) -> Result<(), crate::error::CloseReason> {
648 use crate::error::{CloseReason, LimitKind, OsErrorClass};
649 let Some((file, offset, left)) = self.front_file() else {
650 return Ok(());
651 };
652 let Some(mut b) = self.take_buffer() else {
653 return Err(CloseReason::Limit(LimitKind::BufferBudget));
654 };
655 let spare = b.spare();
656 let take = usize::try_from(left).unwrap_or(usize::MAX).min(spare.len());
657 let read = crate::sys::read_file_at(&file, &mut spare[..take], offset);
658 let n = match read {
659 Ok(0) => {
660 self.release_chunk(b);
661 return Err(CloseReason::Io(OsErrorClass::FileTruncated));
662 }
663 Ok(n) => n,
664 Err(e) => {
665 self.release_chunk(b);
666 return Err(CloseReason::Io(OsErrorClass::from_io(&e)));
667 }
668 };
669 b.set_len(n);
670 self.used += OUTPUT_CLASS.bytes();
671 if let Some(front) = self.chunks.front_mut() {
672 front.advance(n);
673 if front.len() == 0 {
674 self.chunks.pop_front();
675 }
676 }
677 self.chunks.push_front(Chunk::Pooled(b, 0));
678 Ok(())
679 }
680
681 pub fn begin_response(&mut self) {
683 self.response_open = true;
684 self.response_tag = None;
685 self.bytes_since_begin = 0;
686 }
687
688 pub fn begin_tagged_response(&mut self, tag: GateTag) {
690 self.begin_response();
691 self.response_tag = Some(tag);
692 }
693
694 pub fn end_response(&mut self) {
696 self.response_open = false;
697 self.response_tag = None;
698 }
699
700 #[must_use]
702 pub fn pending(&self) -> usize {
703 self.chunks.iter().map(Chunk::len).sum()
704 }
705
706 #[must_use]
708 pub fn is_empty(&self) -> bool {
709 self.chunks.is_empty()
710 }
711
712 pub fn wipe_all(&mut self) {
714 while let Some(chunk) = self.chunks.pop_front() {
715 if let Chunk::Pooled(mut b, _) = chunk {
716 b.wipe();
717 self.release_chunk(b);
718 }
719 }
720 self.used = 0;
721 }
722
723 fn release_chunk(&mut self, mut b: Buffer) {
724 self.used = self
725 .used
726 .saturating_sub(u32::try_from(b.capacity()).unwrap_or(u32::MAX));
727 if self.wipe {
728 b.wipe();
729 } else {
730 b.clear();
731 }
732 if b.class() == OUTPUT_CLASS && self.free.len() < FREE_MAX {
733 self.free.push(b);
734 } else {
735 self.budget
737 .fetch_add(b.capacity() as u64, Ordering::Release);
738 }
739 }
740
741 pub(crate) fn return_spares(&mut self, pool: &mut BufferPool) {
745 while self.free.len() > 1 {
746 if let Some(b) = self.free.pop() {
747 pool.put(b);
748 }
749 }
750 }
751
752 #[cfg(feature = "reactor")]
757 pub(crate) fn return_all_spares(&mut self, pool: &mut BufferPool) {
758 while let Some(b) = self.free.pop() {
759 pool.put(b);
760 }
761 }
762
763 #[cfg(feature = "reactor")]
765 pub(crate) fn pending_in_send(&self) -> usize {
766 self.chunks
767 .iter()
768 .take(crate::sys::MAX_SLICES)
769 .take_while(|c| !c.is_file())
770 .map(Chunk::len)
771 .sum()
772 }
773
774 #[cfg(feature = "reactor")]
776 pub(crate) fn fill_slices<'a>(&'a self, out: &mut [std::io::IoSlice<'a>]) -> usize {
777 let mut n = 0;
778 for chunk in self
779 .chunks
780 .iter()
781 .take(out.len())
782 .take_while(|c| !c.is_file())
783 {
784 out[n] = std::io::IoSlice::new(chunk.remaining());
785 n += 1;
786 }
787 n
788 }
789
790 pub(crate) fn advance(&mut self, mut n: usize) {
792 while let Some(front) = self.chunks.front_mut() {
793 let rem = front.len();
794 if n < rem {
795 front.advance(n);
796 return;
797 }
798 n -= rem;
799 if let Some(Chunk::Pooled(b, _)) = self.chunks.pop_front() {
800 self.release_chunk(b);
801 }
802 }
803 }
804
805 pub(crate) fn reset(&mut self) {
807 self.wipe_all();
808 self.response_open = false;
809 self.response_tag = None;
810 self.bytes_since_begin = 0;
811 self.refused = false;
812 }
813}
814
815pub struct SessionContext<'a> {
817 pub(crate) link: &'a mut Link,
818 pub(crate) output: &'a mut Output,
819 pub(crate) pool: &'a mut BufferPool,
820 pub(crate) completer: &'a Completer,
821 pub(crate) draining: bool,
822 pub(crate) pressure: f32,
823 pub(crate) worker: WorkerId,
824 pub(crate) grow_to: Option<SizeClass>,
825}
826
827impl SessionContext<'_> {
828 #[must_use]
830 pub fn info(&self) -> &ConnectionInfo {
831 &self.link.info
832 }
833
834 pub fn output(&mut self) -> &mut Output {
836 if self.output.free.len() < FREE_MAX {
839 self.output.refill(self.pool);
840 }
841 self.output
842 }
843
844 #[must_use]
846 pub fn now(&self) -> Instant {
847 self.link.now()
848 }
849
850 pub fn set_phase(&mut self, phase: crate::session::Phase) {
852 self.link.set_phase(phase);
853 }
854
855 pub fn set_exchange_deadline(&mut self, at: Option<Instant>) {
857 self.link.set_exchange_deadline(at);
858 }
859
860 pub fn set_idle_timeout(&mut self, timeout: Option<Duration>) {
881 self.link.set_idle_timeout(timeout);
882 }
883
884 #[must_use]
886 pub fn draining(&self) -> bool {
887 self.draining
888 }
889
890 #[must_use]
892 pub fn pressure(&self) -> f32 {
893 self.pressure
894 }
895
896 #[must_use]
898 pub fn kill_handle(&self) -> KillHandle {
899 KillHandle {
900 slot: self.link.slot.clone(),
901 generation: self.link.generation,
902 id: self.link.info.id,
903 }
904 }
905
906 #[must_use]
910 pub fn completer(&self) -> Completer {
911 self.link.wakeable.set(true);
912 self.completer.clone()
913 }
914
915 #[must_use]
917 pub fn worker(&self) -> WorkerId {
918 self.worker
919 }
920
921 pub fn buffers(&mut self) -> &mut BufferPool {
923 self.pool
924 }
925
926 pub fn release(&mut self, buffer: Buffer) {
928 self.pool.put(buffer);
929 }
930
931 pub fn grow_input(&mut self, class: SizeClass) -> Result<(), NetError> {
937 if class > self.link.limits.read_class_max {
938 return Err(NetError::Limit {
939 which: LimitKind::ReadBuffer,
940 });
941 }
942 self.grow_to = Some(class);
943 Ok(())
944 }
945}
946
947impl fmt::Debug for SessionContext<'_> {
948 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
949 f.debug_struct("SessionContext")
950 .field("id", &self.link.info.id)
951 .field("draining", &self.draining)
952 .finish()
953 }
954}
955
956#[cfg(test)]
957mod tests {
958 use super::*;
959
960 #[test]
961 fn test_input_consume_saturates_and_resets() {
962 let mut i = Input::from_bytes(b"abc");
963 i.consume(10);
964 assert!(i.bytes().is_empty());
965 assert_eq!(i.start, 0);
966 }
967
968 #[test]
969 fn test_input_swap_rules() {
970 let mut i = Input::from_bytes(b"xyz");
971 let mut pool = BufferPool::standalone(1 << 22);
972 let mut wrong = pool.take(SizeClass::LARGE).unwrap();
973 assert_eq!(i.swap(&mut wrong), Err(SwapError::ClassMismatch));
974 let mut full = pool.take(i.class()).unwrap();
975 full.set_len(1);
976 assert_eq!(i.swap(&mut full), Err(SwapError::NotEmpty));
977 let mut empty = pool.take(i.class()).unwrap();
978 i.consume(1);
979 i.swap(&mut empty).unwrap();
980 assert_eq!(empty.as_slice(), b"yz");
981 }
982
983 #[test]
984 fn test_output_push_all_or_nothing_and_partial() {
985 let mut o = Output::standalone(OUTPUT_CLASS.bytes());
986 let big = vec![7u8; OUTPUT_CLASS.bytes() as usize + 1];
987 assert_eq!(o.push(&big), Err(OutputFull));
988 assert!(o.is_empty());
989 assert_eq!(o.push_partial(&big), OUTPUT_CLASS.bytes() as usize);
990 assert_eq!(o.pending(), OUTPUT_CLASS.bytes() as usize);
991 }
992
993 #[test]
994 fn test_output_advance_releases_chunks() {
995 let mut o = Output::standalone(1 << 20);
996 o.push(b"hello").unwrap();
997 o.push_static(b" world").unwrap();
998 o.advance(7);
999 assert_eq!(o.pending(), 4);
1000 assert_eq!(o.chunks.len(), 1);
1001 assert_eq!(o.used, 0);
1002 o.advance(4);
1003 assert!(o.is_empty());
1004 }
1005
1006 #[test]
1007 fn test_output_response_markers() {
1008 let mut o = Output::standalone(1 << 20);
1009 o.begin_tagged_response(GateTag(3));
1010 assert_eq!(o.response_tag, Some(GateTag(3)));
1011 o.end_response();
1012 assert_eq!(o.response_tag, None);
1013 }
1014
1015 #[test]
1018 fn prop_output_stream_matches_accepted_pushes() {
1019 use crate::testing::Gen;
1020 static TEXT: [u8; 4096] = {
1021 let mut t = [0u8; 4096];
1022 let mut i = 0;
1023 while i < t.len() {
1024 t[i] = (i % 251) as u8;
1025 i += 1;
1026 }
1027 t
1028 };
1029 let mut g = Gen::new(0x0047_0041);
1030 for round in 0..if cfg!(miri) { 3 } else { 300 } {
1031 let mut o = Output::standalone(64 * 1024);
1032 let mut expected = Vec::new();
1033 let mut sent = Vec::new();
1034 for _ in 0..200 {
1035 match g.below(4) {
1036 0 => {
1037 let n = g.below(40_000) as usize;
1038 let data: Vec<u8> = (0..n).map(|_| g.next_u64() as u8).collect();
1039 let took = o.push_partial(&data);
1040 expected.extend_from_slice(&data[..took]);
1041 }
1042 1 => {
1043 let n = g.below(3000) as usize;
1044 let data: Vec<u8> = (0..n).map(|_| g.next_u64() as u8).collect();
1045 if o.push(&data).is_ok() {
1046 expected.extend_from_slice(&data);
1047 }
1048 }
1049 2 => {
1050 let start = g.below(TEXT.len() as u64) as usize;
1051 let piece = &TEXT[start..];
1052 if o.push_static(piece).is_ok() {
1053 expected.extend_from_slice(piece);
1054 }
1055 }
1056 _ => {
1057 let mut want = g.below(70_000) as usize;
1059 let mut taken = 0;
1060 for chunk in o.chunks.iter() {
1061 if want == 0 {
1062 break;
1063 }
1064 let r = chunk.remaining();
1065 let n = r.len().min(want);
1066 sent.extend_from_slice(&r[..n]);
1067 want -= n;
1068 taken += n;
1069 }
1070 o.advance(taken);
1071 }
1072 }
1073 }
1074 while !o.is_empty() {
1075 let n = o.chunks.front().map_or(0, |c| c.remaining().len());
1076 if let Some(c) = o.chunks.front() {
1077 sent.extend_from_slice(c.remaining());
1078 }
1079 o.advance(n);
1080 }
1081 assert!(sent == expected, "round {round}: stream differs");
1082 assert_eq!(o.used, 0, "round {round}: budget not returned");
1083 assert_eq!(o.pending(), 0);
1084 }
1085 }
1086}