1use core::fmt;
32use std::collections::BTreeMap;
33
34use astrs_wire::{DataId, Metadata};
35
36use crate::error::{NodeError, Result};
37use crate::events::Event;
38use crate::node::Node;
39use crate::output::RawOutput;
40use crate::payload::Payload;
41
42#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
44pub struct ChunkRef {
45 pub session: String,
47 pub segment: i64,
49 pub seq: i64,
51 pub fin: bool,
53 pub flush: bool,
55}
56
57impl ChunkRef {
58 #[must_use]
60 pub fn from_metadata(metadata: &Metadata) -> Option<Self> {
61 Some(Self {
62 session: metadata.session_id()?.to_owned(),
63 segment: metadata.segment_id().unwrap_or(0),
64 seq: metadata.seq().unwrap_or(0),
65 fin: metadata.fin().unwrap_or(false),
66 flush: metadata.flush().unwrap_or(false),
67 })
68 }
69
70 #[must_use]
72 pub fn segment_key(&self) -> (String, i64) {
73 (self.session.clone(), self.segment)
74 }
75}
76
77impl fmt::Display for ChunkRef {
78 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
79 write!(f, "{}/{}#{}", self.session, self.segment, self.seq)?;
80 if self.fin {
81 f.write_str(" fin")?;
82 }
83 if self.flush {
84 f.write_str(" flush")?;
85 }
86 Ok(())
87 }
88}
89
90pub type AcceptedChunk = core::result::Result<Option<StreamSegment>, Box<Event>>;
94
95#[derive(Debug, Clone, PartialEq, Eq)]
97pub struct StreamSegment {
98 pub session: String,
100 pub segment: i64,
102 pub bytes: Vec<u8>,
104 pub chunks: usize,
106}
107
108#[derive(Debug, Clone)]
113pub struct StreamWriter {
114 session: String,
116 segment: i64,
118 seq: i64,
120 finished: u64,
122}
123
124impl StreamWriter {
125 #[must_use]
127 pub fn new() -> Self {
128 Self::with_session(super::fresh_id())
129 }
130
131 #[must_use]
133 pub fn with_session(session: impl Into<String>) -> Self {
134 Self {
135 session: session.into(),
136 segment: 0,
137 seq: 0,
138 finished: 0,
139 }
140 }
141
142 #[must_use]
144 pub fn session(&self) -> &str {
145 &self.session
146 }
147
148 #[must_use]
150 pub const fn segment(&self) -> i64 {
151 self.segment
152 }
153
154 #[must_use]
156 pub const fn seq(&self) -> i64 {
157 self.seq
158 }
159
160 #[must_use]
162 pub const fn finished_segments(&self) -> u64 {
163 self.finished
164 }
165
166 pub fn next_metadata(&mut self, base: Metadata, fin: bool, flush: bool) -> Metadata {
171 let mut metadata = base;
172 metadata.set_session_id(self.session.clone());
173 metadata.set_segment_id(self.segment);
174 metadata.set_seq(self.seq);
175 if fin {
176 metadata.set_fin(true);
177 }
178 if flush {
179 metadata.set_flush(true);
180 }
181 if fin {
182 self.segment = self.segment.saturating_add(1);
183 self.seq = 0;
184 self.finished = self.finished.saturating_add(1);
185 } else {
186 self.seq = self.seq.saturating_add(1);
187 }
188 metadata
189 }
190
191 pub fn abandon_segment(&mut self) {
197 self.segment = self.segment.saturating_add(1);
198 self.seq = 0;
199 }
200}
201
202impl Default for StreamWriter {
203 fn default() -> Self {
204 Self::new()
205 }
206}
207
208#[derive(Debug, Default)]
210pub struct StreamAssembler {
211 open: BTreeMap<(String, i64), Partial>,
213 completed: u64,
215}
216
217#[derive(Debug, Default)]
219struct Partial {
220 bytes: Vec<u8>,
222 next_seq: i64,
224 chunks: usize,
226}
227
228impl StreamAssembler {
229 #[must_use]
231 pub fn new() -> Self {
232 Self::default()
233 }
234
235 #[must_use]
237 pub fn open_segments(&self) -> Vec<(String, i64)> {
238 self.open.keys().cloned().collect()
239 }
240
241 #[must_use]
243 pub const fn completed(&self) -> u64 {
244 self.completed
245 }
246
247 #[must_use]
249 pub fn buffered_bytes(&self) -> usize {
250 self.open.values().map(|partial| partial.bytes.len()).sum()
251 }
252
253 pub fn accept(&mut self, chunk: &ChunkRef, bytes: &[u8]) -> Result<Option<StreamSegment>> {
261 let key = chunk.segment_key();
262 let partial = self.open.entry(key.clone()).or_default();
263 if chunk.seq != partial.next_seq {
264 let expected = partial.next_seq;
265 let _abandoned = self.open.remove(&key);
266 return Err(NodeError::Pattern(format!(
267 "stream {}/{}: expected chunk {expected}, got {}",
268 chunk.session, chunk.segment, chunk.seq
269 )));
270 }
271 partial.bytes.extend_from_slice(bytes);
272 partial.next_seq = partial.next_seq.saturating_add(1);
273 partial.chunks += 1;
274 if !chunk.fin {
275 return Ok(None);
276 }
277 let Some(partial) = self.open.remove(&key) else {
278 return Ok(None);
279 };
280 self.completed = self.completed.saturating_add(1);
281 Ok(Some(StreamSegment {
282 session: chunk.session.clone(),
283 segment: chunk.segment,
284 bytes: partial.bytes,
285 chunks: partial.chunks,
286 }))
287 }
288
289 pub fn accept_event(&mut self, event: Event) -> Result<AcceptedChunk> {
299 let Event::Input { id, meta, data } = event else {
300 return Ok(Err(Box::new(event)));
301 };
302 let Some(chunk) = ChunkRef::from_metadata(&meta) else {
303 return Ok(Err(Box::new(Event::Input { id, meta, data })));
304 };
305 self.accept(&chunk, data.bytes()).map(Ok)
306 }
307
308 pub fn abandon(&mut self, session: &str, segment: i64) -> Option<Vec<u8>> {
312 self.open
313 .remove(&(session.to_owned(), segment))
314 .map(|partial| partial.bytes)
315 }
316}
317
318impl Node {
319 pub fn stream_chunk(
328 &self,
329 output: &mut RawOutput,
330 writer: &mut StreamWriter,
331 payload: impl AsRef<[u8]>,
332 fin: bool,
333 ) -> Result<ChunkRef> {
334 self.stream_chunk_with(output, writer, payload, fin, false)
335 }
336
337 pub fn stream_chunk_with(
344 &self,
345 output: &mut RawOutput,
346 writer: &mut StreamWriter,
347 payload: impl AsRef<[u8]>,
348 fin: bool,
349 flush: bool,
350 ) -> Result<ChunkRef> {
351 let metadata = writer.next_metadata(self.metadata(), fin, flush);
352 let chunk = ChunkRef::from_metadata(&metadata)
353 .ok_or_else(|| NodeError::Pattern("a stream chunk needs a session id".to_owned()))?;
354 output.send_bytes(payload, metadata)?;
355 Ok(chunk)
356 }
357
358 pub fn stream_segment(
366 &self,
367 output: &mut RawOutput,
368 writer: &mut StreamWriter,
369 payload: &[u8],
370 chunk_size: usize,
371 ) -> Result<usize> {
372 if chunk_size == 0 {
373 return Err(NodeError::Pattern(
374 "a stream chunk size of zero would never finish".to_owned(),
375 ));
376 }
377 if payload.is_empty() {
378 let _chunk = self.stream_chunk(output, writer, [], true)?;
379 return Ok(1);
380 }
381 let mut sent = 0;
382 let mut chunks = payload.chunks(chunk_size).peekable();
383 while let Some(chunk) = chunks.next() {
384 let last = chunks.peek().is_none();
385 let _reference = self.stream_chunk(output, writer, chunk, last)?;
386 sent += 1;
387 }
388 Ok(sent)
389 }
390
391 #[must_use]
393 pub fn stream_chunk_of(event: &Event) -> Option<(DataId, ChunkRef)> {
394 let Event::Input { id, meta, .. } = event else {
395 return None;
396 };
397 ChunkRef::from_metadata(meta).map(|chunk| (id.clone(), chunk))
398 }
399}
400
401#[must_use]
403pub fn chunk_bytes(event: &Event) -> Option<&[u8]> {
404 event.payload().map(Payload::bytes)
405}
406
407#[cfg(test)]
408mod tests {
409 #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
410
411 use super::*;
412 use crate::testing::TestHarness;
413 use astrs_time::HlcTimestamp;
414 use std::time::Duration;
415
416 fn metadata() -> Metadata {
417 Metadata::new(HlcTimestamp::new(1, 0))
418 }
419
420 #[test]
421 fn a_writer_numbers_its_chunks_and_segments() {
422 let mut writer = StreamWriter::with_session("s1");
423 assert_eq!(writer.session(), "s1");
424 assert_eq!(writer.segment(), 0);
425 assert_eq!(writer.seq(), 0);
426
427 let first = writer.next_metadata(metadata(), false, false);
428 assert_eq!(first.seq(), Some(0));
429 assert_eq!(first.segment_id(), Some(0));
430 assert_eq!(writer.seq(), 1);
431
432 let second = writer.next_metadata(metadata(), true, false);
433 assert_eq!(second.seq(), Some(1));
434 assert_eq!(second.fin(), Some(true));
435 assert_eq!(writer.segment(), 1, "fin starts the next segment");
436 assert_eq!(writer.seq(), 0);
437 assert_eq!(writer.finished_segments(), 1);
438
439 writer.abandon_segment();
440 assert_eq!(writer.segment(), 2);
441 assert_eq!(writer.finished_segments(), 1, "abandoning is not finishing");
442 }
443
444 #[test]
445 fn a_flush_is_carried_without_ending_the_segment() {
446 let mut writer = StreamWriter::new();
447 let metadata = writer.next_metadata(metadata(), false, true);
448 assert_eq!(metadata.flush(), Some(true));
449 assert_eq!(metadata.fin(), None);
450 assert_eq!(writer.segment(), 0);
451 }
452
453 #[test]
454 fn an_assembler_reassembles_a_segment() {
455 let mut writer = StreamWriter::with_session("s1");
456 let mut assembler = StreamAssembler::new();
457
458 let first =
459 ChunkRef::from_metadata(&writer.next_metadata(metadata(), false, false)).unwrap();
460 assert!(assembler.accept(&first, b"hello ").unwrap().is_none());
461 assert_eq!(assembler.open_segments(), vec![("s1".to_owned(), 0)]);
462 assert_eq!(assembler.buffered_bytes(), 6);
463
464 let last = ChunkRef::from_metadata(&writer.next_metadata(metadata(), true, false)).unwrap();
465 let segment = assembler.accept(&last, b"world").unwrap().unwrap();
466 assert_eq!(segment.bytes, b"hello world");
467 assert_eq!(segment.chunks, 2);
468 assert_eq!(segment.segment, 0);
469 assert_eq!(segment.session, "s1");
470 assert_eq!(assembler.completed(), 1);
471 assert!(assembler.open_segments().is_empty());
472 assert_eq!(assembler.buffered_bytes(), 0);
473 }
474
475 #[test]
476 fn a_gap_is_reported_rather_than_reassembled_wrong() {
477 let mut assembler = StreamAssembler::new();
478 let first = ChunkRef {
479 session: "s".to_owned(),
480 segment: 0,
481 seq: 0,
482 fin: false,
483 flush: false,
484 };
485 assert!(assembler.accept(&first, b"a").unwrap().is_none());
486
487 let skipped = ChunkRef { seq: 2, ..first };
488 let error = assembler.accept(&skipped, b"c").unwrap_err();
489 assert!(matches!(error, NodeError::Pattern(_)), "{error}");
490 assert!(error.to_string().contains("expected chunk 1"), "{error}");
491 assert!(
492 assembler.open_segments().is_empty(),
493 "a broken segment is abandoned, not left half-built"
494 );
495 }
496
497 #[test]
498 fn interleaved_segments_are_kept_apart() {
499 let mut assembler = StreamAssembler::new();
500 let a0 = ChunkRef {
501 session: "s".to_owned(),
502 segment: 0,
503 seq: 0,
504 fin: false,
505 flush: false,
506 };
507 let b0 = ChunkRef {
508 segment: 1,
509 ..a0.clone()
510 };
511 assert!(assembler.accept(&a0, b"A").unwrap().is_none());
512 assert!(assembler.accept(&b0, b"B").unwrap().is_none());
513 assert_eq!(assembler.open_segments().len(), 2);
514
515 let a1 = ChunkRef {
516 seq: 1,
517 fin: true,
518 ..a0
519 };
520 let segment = assembler.accept(&a1, b"a").unwrap().unwrap();
521 assert_eq!(segment.bytes, b"Aa");
522 assert_eq!(assembler.open_segments(), vec![("s".to_owned(), 1)]);
523 assert_eq!(assembler.abandon("s", 1).unwrap(), b"B");
524 assert!(assembler.abandon("s", 9).is_none());
525 }
526
527 #[test]
528 fn a_non_stream_event_is_handed_back() {
529 let mut assembler = StreamAssembler::new();
530 let event = Event::Input {
531 id: DataId::new("in").unwrap(),
532 meta: metadata(),
533 data: Payload::inline(vec![1]),
534 };
535 let handed_back = assembler.accept_event(event).unwrap().unwrap_err();
536 assert!(handed_back.is_input());
537 assert!(chunk_bytes(&handed_back).is_some());
538
539 let control = assembler
540 .accept_event(Event::AllInputsClosed)
541 .unwrap()
542 .unwrap_err();
543 assert!(!control.is_input());
544 assert!(chunk_bytes(&control).is_none());
545 }
546
547 #[test]
548 fn a_whole_payload_is_chunked_and_reassembled_over_the_wire() {
549 let (mut producer, mut consumer) =
550 TestHarness::pair("sender", "chunks", "receiver", "chunks").unwrap();
551 let mut output = producer.node.raw_output("chunks").unwrap();
552 let mut writer = StreamWriter::with_session("transfer-1");
553
554 let payload: Vec<u8> = (0..250u8).collect();
555 let sent = producer
556 .node
557 .stream_segment(&mut output, &mut writer, &payload, 64)
558 .unwrap();
559 assert_eq!(sent, 4, "250 bytes in 64-byte chunks");
560
561 let mut assembler = StreamAssembler::new();
562 let mut segment = None;
563 for _ in 0..sent {
564 let event = consumer
565 .events
566 .recv_timeout(Duration::from_secs(5))
567 .unwrap()
568 .expect("a chunk");
569 assert!(Node::stream_chunk_of(&event).is_some());
570 if let Ok(Some(done)) = assembler.accept_event(event).unwrap() {
571 segment = Some(done);
572 }
573 }
574 let segment = segment.expect("the reassembled segment");
575 assert_eq!(segment.bytes, payload);
576 assert_eq!(segment.chunks, 4);
577 assert_eq!(segment.session, "transfer-1");
578 }
579
580 #[test]
581 fn an_empty_payload_still_sends_one_terminating_chunk() {
582 let mut harness = TestHarness::start().unwrap();
583 let mut output = harness
584 .node
585 .raw_output(TestHarness::DEFAULT_OUTPUT)
586 .unwrap();
587 let mut writer = StreamWriter::new();
588 let sent = harness
589 .node
590 .stream_segment(&mut output, &mut writer, &[], 32)
591 .unwrap();
592 assert_eq!(sent, 1);
593
594 let sends = harness
595 .daemon
596 .wait_for_sends(
597 harness.node.id(),
598 &DataId::new(TestHarness::DEFAULT_OUTPUT).unwrap(),
599 1,
600 Duration::from_secs(5),
601 )
602 .unwrap();
603 assert_eq!(sends[0].metadata.fin(), Some(true));
604 assert_eq!(sends[0].payload.len(), 0);
605 }
606
607 #[test]
608 fn a_zero_chunk_size_is_refused() {
609 let mut harness = TestHarness::start().unwrap();
610 let mut output = harness
611 .node
612 .raw_output(TestHarness::DEFAULT_OUTPUT)
613 .unwrap();
614 let mut writer = StreamWriter::new();
615 let error = harness
616 .node
617 .stream_segment(&mut output, &mut writer, b"data", 0)
618 .unwrap_err();
619 assert!(matches!(error, NodeError::Pattern(_)), "{error}");
620 }
621
622 #[test]
623 fn chunk_references_render_their_place() {
624 let chunk = ChunkRef {
625 session: "s".to_owned(),
626 segment: 2,
627 seq: 3,
628 fin: true,
629 flush: true,
630 };
631 assert_eq!(chunk.to_string(), "s/2#3 fin flush");
632 assert_eq!(chunk.segment_key(), ("s".to_owned(), 2));
633 assert_eq!(ChunkRef::from_metadata(&metadata()), None);
634 }
635}