use crate::Context;
use core::time::Duration;
use std::collections::VecDeque;
use super::{Group, Joins, Pairing};
use crate::{
Arm, Custody, Delivery, Drives, Emitter, Inbox, Intake, Item, ItemId, Operator, Owing, Payload,
Resident, Share, Wake, ack_channel, arm, channel, park_on, select,
};
#[derive(Debug, Clone, PartialEq, Eq)]
struct Frame {
media: u64,
at: u64,
}
#[derive(Debug, Clone, PartialEq, Eq)]
struct Segment {
media: u64,
said: &'static str,
}
#[derive(Debug, Clone, PartialEq, Eq)]
struct Scene {
media: u64,
}
#[derive(Debug, Clone, PartialEq, Eq)]
enum CaptionIn {
Paired(Frame, Segment),
Scene(Scene),
}
#[derive(Debug, Default)]
struct CaptionGroup {
frame: Option<Frame>,
segment: Option<Segment>,
}
impl Group for CaptionGroup {
type Complete = (Frame, Segment);
fn complete(self) -> Result<Self::Complete, Self> {
match (self.frame, self.segment) {
(Some(frame), Some(segment)) => Ok((frame, segment)),
(frame, segment) => Err(Self { frame, segment }),
}
}
}
struct ByMedia;
impl Joins<Frame> for ByMedia {
type Key = u64;
fn key(&self, message: &Frame) -> u64 {
message.media
}
}
impl Joins<Segment> for ByMedia {
type Key = u64;
fn key(&self, message: &Segment) -> u64 {
message.media
}
}
enum Taken {
Frames(Wake<ItemId, Frame>),
Segments(Wake<ItemId, Segment>),
Scenes(Wake<ItemId, Scene>),
}
struct CaptionIntake {
frames: Inbox<ItemId, Frame>,
segments: Inbox<ItemId, Segment>,
scenes: Inbox<ItemId, Scene>,
joins: ByMedia,
pairing: Pairing<u64, CaptionGroup>,
ready: VecDeque<Wake<ItemId, CaptionIn>>,
open: [bool; Self::UPSTREAMS],
turn: usize,
next: u64,
}
impl CaptionIntake {
const UPSTREAMS: usize = 3;
fn new(
frames: Inbox<ItemId, Frame>,
segments: Inbox<ItemId, Segment>,
scenes: Inbox<ItemId, Scene>,
) -> Self {
Self {
frames,
segments,
scenes,
joins: ByMedia,
pairing: Pairing::new(),
ready: VecDeque::new(),
open: [true; Self::UPSTREAMS],
turn: 0,
next: 0,
}
}
fn joined(
&mut self,
(frame, segment): (Frame, Segment),
shares: Vec<Share<ItemId>>,
) -> Wake<ItemId, CaptionIn> {
let key = ItemId::new(self.next);
self.next += 1;
Wake::Delivered(Delivery::joined(
Payload::One(Item::new(key, CaptionIn::Paired(frame, segment))),
Custody::joined(key, shares),
))
}
fn keyed<T>(&mut self, delivery: Delivery<T>, fill: impl Fn(&mut CaptionGroup, T))
where
ByMedia: Joins<T, Key = u64>,
{
let (payload, custody) = delivery.into_parts();
let owing = Owing::open(payload.keys().copied(), custody);
for item in payload {
let (key, message) = item.into_parts();
let join = self.joins.key(&message);
let completed = self
.pairing
.arrive(join, owing.share(key), |group| fill(group, message));
if let Some((pair, shares)) = completed {
let delivered = self.joined(pair, shares);
self.ready.push_back(delivered);
}
}
}
fn take(&mut self, taken: Taken) {
match taken {
Taken::Frames(Wake::Delivered(delivery)) => {
self.keyed(delivery, |group, frame| group.frame = Some(frame));
}
Taken::Segments(Wake::Delivered(delivery)) => {
self.keyed(delivery, |group, segment| group.segment = Some(segment));
}
Taken::Scenes(Wake::Delivered(delivery)) => self
.ready
.push_back(Wake::Delivered(delivery.map(CaptionIn::Scene))),
Taken::Frames(Wake::Sweep) | Taken::Segments(Wake::Sweep) | Taken::Scenes(Wake::Sweep) => {
self.ready.push_back(Wake::Sweep);
}
Taken::Frames(Wake::Tick(tick))
| Taken::Segments(Wake::Tick(tick))
| Taken::Scenes(Wake::Tick(tick)) => self.ready.push_back(Wake::Tick(tick)),
}
}
}
impl Intake<CaptionIn> for CaptionIntake {
fn park(&mut self, sweep: Duration) -> Option<Wake<ItemId, CaptionIn>> {
park_on(self.park_async(), sweep).unwrap_or(Some(Wake::Sweep))
}
async fn park_async(&mut self) -> Option<Wake<ItemId, CaptionIn>> {
loop {
if let Some(wake) = self.ready.pop_front() {
return Some(wake);
}
if self.open.iter().all(|open| !open) {
return None;
}
let answered = {
let Self {
frames,
segments,
scenes,
open,
turn,
..
} = &*self;
let mut first = core::pin::pin!(arm(open[0], frames, Taken::Frames));
let mut second = core::pin::pin!(arm(open[1], segments, Taken::Segments));
let mut third = core::pin::pin!(arm(open[2], scenes, Taken::Scenes));
let mut arms: [Arm<'_, Option<Taken>>; Self::UPSTREAMS] =
[first.as_mut(), second.as_mut(), third.as_mut()];
select(*turn, &mut arms).await
};
let (index, taken) = answered?;
self.turn = (index + 1) % Self::UPSTREAMS;
match taken {
Some(taken) => self.take(taken),
None => self.open[index] = false,
}
}
}
fn drain(&mut self, into: &mut Vec<Wake<ItemId, CaptionIn>>) {
for wake in self.frames.drain().collect::<Vec<_>>() {
self.take(Taken::Frames(wake));
}
for wake in self.segments.drain().collect::<Vec<_>>() {
self.take(Taken::Segments(wake));
}
for wake in self.scenes.drain().collect::<Vec<_>>() {
self.take(Taken::Scenes(wake));
}
into.extend(self.ready.drain(..));
}
}
struct Wired {
frames: crate::Outbox<ItemId, Frame>,
segments: crate::Outbox<ItemId, Segment>,
scenes: crate::Outbox<ItemId, Scene>,
intake: CaptionIntake,
}
fn wired() -> Wired {
let (frames, frames_in) = channel::<ItemId, Frame>(8);
let (segments, segments_in) = channel::<ItemId, Segment>(8);
let (scenes, scenes_in) = channel::<ItemId, Scene>(8);
Wired {
frames,
segments,
scenes,
intake: CaptionIntake::new(frames_in, segments_in, scenes_in),
}
}
fn send<T>(outbox: &crate::Outbox<ItemId, T>, items: Vec<(u64, T)>) -> crate::AckReceiver<ItemId> {
let (ack, answers) = ack_channel::<ItemId>();
let batch = items
.into_iter()
.map(|(key, value)| Item::new(ItemId::new(key), value))
.collect();
outbox
.deliver(Delivery::new(Payload::Batch(batch), ack))
.map_err(|_| "the consumer is here")
.expect("delivered");
answers
}
fn parked(intake: &mut CaptionIntake) -> Option<Wake<ItemId, CaptionIn>> {
match intake.park(Duration::from_millis(60)) {
Some(Wake::Sweep) | None => None,
other => other,
}
}
fn input(wake: Wake<ItemId, CaptionIn>) -> CaptionIn {
let Wake::Delivered(delivery) = wake else {
panic!("a delivery");
};
delivery.into_items().next().expect("one item").into_value()
}
#[test]
fn two_keyed_streams_join_on_key_whichever_arrives_first() {
let mut world = wired();
let _frames = send(&world.frames, vec![(0, Frame { media: 7, at: 1 })]);
assert!(
parked(&mut world.intake).is_none(),
"a lone member is not a pair"
);
let _segments = send(
&world.segments,
vec![(
0,
Segment {
media: 7,
said: "hello",
},
)],
);
assert_eq!(
input(parked(&mut world.intake).expect("the pair")),
CaptionIn::Paired(
Frame { media: 7, at: 1 },
Segment {
media: 7,
said: "hello"
}
)
);
let _late = send(
&world.segments,
vec![(
1,
Segment {
media: 9,
said: "again",
},
)],
);
assert!(
parked(&mut world.intake).is_none(),
"the other order waits the same way"
);
let _frames = send(&world.frames, vec![(1, Frame { media: 9, at: 2 })]);
assert_eq!(
input(parked(&mut world.intake).expect("the pair")),
CaptionIn::Paired(
Frame { media: 9, at: 2 },
Segment {
media: 9,
said: "again"
}
)
);
}
#[test]
fn an_unpaired_key_waits_while_every_other_key_flows() {
let mut world = wired();
let _frames = send(
&world.frames,
vec![
(0, Frame { media: 1, at: 1 }),
(1, Frame { media: 2, at: 2 }),
(2, Frame { media: 3, at: 3 }),
],
);
let _segments = send(
&world.segments,
vec![
(
0,
Segment {
media: 3,
said: "three",
},
),
(
1,
Segment {
media: 2,
said: "two",
},
),
],
);
let mut paired = Vec::new();
while let Some(wake) = parked(&mut world.intake) {
let CaptionIn::Paired(frame, _) = input(wake) else {
panic!("a pair");
};
paired.push(frame.media);
}
paired.sort_unstable();
assert_eq!(
paired,
[2, 3],
"the two keys that completed, and only those"
);
assert_eq!(
world.intake.pairing.waiting(),
1,
"media 1 is still waiting for a segment that never came"
);
}
#[test]
fn a_straight_source_flows_while_a_pair_is_still_half_arrived() {
let mut world = wired();
let _frames = send(&world.frames, vec![(0, Frame { media: 4, at: 1 })]);
let _scenes = send(&world.scenes, vec![(0, Scene { media: 4 })]);
assert_eq!(
input(parked(&mut world.intake).expect("the scene")),
CaptionIn::Scene(Scene { media: 4 }),
"an independent flow never waits on a pair that has not assembled"
);
assert!(parked(&mut world.intake).is_none());
assert_eq!(world.intake.pairing.waiting(), 1);
}
#[test]
fn a_straight_source_keeps_its_own_keys_and_its_own_token() {
let mut world = wired();
let answers = send(
&world.scenes,
vec![(3, Scene { media: 5 }), (4, Scene { media: 6 })],
);
let Some(Wake::Delivered(delivery)) = parked(&mut world.intake) else {
panic!("the scenes");
};
let keys: Vec<ItemId> = delivery.payload().keys().copied().collect();
assert_eq!(keys, [ItemId::new(3), ItemId::new(4)]);
let (payload, custody) = delivery.into_parts();
let released = payload.keys().copied().collect();
assert!(
custody
.expect("the token came through")
.send(crate::Ack::new(
released,
"captioner",
crate::Lifecycle::Processed(())
))
);
let crate::Answer::Settled(settled) = answers.take() else {
panic!("answered");
};
assert_eq!(settled.released().len(), 2);
}
#[test]
fn a_pair_s_delivery_carries_both_members_custody_and_releases_both() {
let mut world = wired();
let frames = send(&world.frames, vec![(0, Frame { media: 8, at: 1 })]);
let segments = send(
&world.segments,
vec![(
0,
Segment {
media: 8,
said: "eight",
},
)],
);
let Some(Wake::Delivered(delivery)) = parked(&mut world.intake) else {
panic!("the pair");
};
let (payload, custody) = delivery.into_parts();
let released = payload.keys().copied().collect();
assert!(custody.expect("composed").send(crate::Ack::new(
released,
"captioner",
crate::Lifecycle::Processed(())
)));
for answers in [frames, segments] {
let crate::Answer::Settled(settled) = answers.take() else {
panic!("every member is answered");
};
assert_eq!(settled.released().len(), 1);
assert!(settled.released().contains(&ItemId::new(0)));
assert_eq!(settled.worker().as_str(), "captioner");
}
}
#[test]
fn a_second_message_for_a_filled_slot_replaces_it_and_still_answers_the_first() {
let mut world = wired();
let first = send(&world.frames, vec![(0, Frame { media: 2, at: 10 })]);
let second = send(&world.frames, vec![(1, Frame { media: 2, at: 20 })]);
let mut absorbed = Vec::new();
world.intake.drain(&mut absorbed);
assert!(absorbed.is_empty(), "neither frame is a pair on its own");
let segments = send(
&world.segments,
vec![(
0,
Segment {
media: 2,
said: "two",
},
)],
);
let Some(Wake::Delivered(delivery)) = parked(&mut world.intake) else {
panic!("the pair");
};
let (payload, custody) = delivery.into_parts();
let CaptionIn::Paired(frame, _) = payload.into_iter().next().expect("one item").into_value()
else {
panic!("a pair");
};
assert_eq!(frame.at, 20, "the newer message won the slot");
let mut released = std::collections::HashSet::new();
released.insert(ItemId::new(0));
released.insert(ItemId::new(1));
assert!(custody.expect("composed").send(crate::Ack::new(
released,
"captioner",
crate::Lifecycle::Processed(())
)));
for answers in [first, second, segments] {
let crate::Answer::Settled(settled) = answers.take() else {
panic!("all three deliveries are answered");
};
assert_eq!(settled.released().len(), 1);
}
}
#[test]
fn a_batch_whose_keys_join_different_groups_answers_only_when_the_last_is_decided() {
let mut world = wired();
let frames = send(
&world.frames,
vec![
(0, Frame { media: 1, at: 1 }),
(1, Frame { media: 2, at: 2 }),
(2, Frame { media: 3, at: 3 }),
],
);
let mut released = std::collections::HashSet::new();
for (key, media) in [(0u64, 1u64), (1, 2)] {
let _segments = send(&world.segments, vec![(key, Segment { media, said: "s" })]);
let Some(Wake::Delivered(delivery)) = parked(&mut world.intake) else {
panic!("the pair for media {media}");
};
let (payload, custody) = delivery.into_parts();
released.extend(payload.keys().copied());
let taken = payload.keys().copied().collect();
assert!(custody.expect("composed").send(crate::Ack::new(
taken,
"captioner",
crate::Lifecycle::Processed(())
)));
assert!(
matches!(frames.take(), crate::Answer::Waiting),
"two of three decided says nothing"
);
}
let _segments = send(
&world.segments,
vec![(
2,
Segment {
media: 3,
said: "s",
},
)],
);
let Some(Wake::Delivered(delivery)) = parked(&mut world.intake) else {
panic!("the last pair");
};
let (payload, custody) = delivery.into_parts();
let taken = payload.keys().copied().collect();
assert!(custody.expect("composed").send(crate::Ack::new(
taken,
"captioner",
crate::Lifecycle::Processed(())
)));
let crate::Answer::Settled(settled) = frames.take() else {
panic!("the pass that finished it answers");
};
assert_eq!(
settled.released().len(),
3,
"every item of the batch, each released by the pair it went into"
);
}
#[test]
fn the_intake_is_shut_down_only_when_every_source_is_gone() {
let mut world = wired();
drop(world.frames);
assert!(
world.intake.park(Duration::from_millis(60)).is_some(),
"one upstream gone is not shutdown"
);
drop(world.segments);
assert!(world.intake.park(Duration::from_millis(60)).is_some());
drop(world.scenes);
assert!(
world.intake.park(Duration::from_secs(1)).is_none(),
"every upstream gone is shutdown, which is what ends the loop"
);
}
struct Captioner;
impl Operator for Captioner {
type Error = core::convert::Infallible;
type In = CaptionIn;
type Out = String;
fn accept(
&mut self,
_ctx: &Context,
delivery: Delivery<CaptionIn>,
emitter: &mut Emitter<String>,
) -> Result<(), Self::Error> {
for item in delivery.into_items() {
let caption = match item.into_value() {
CaptionIn::Paired(frame, segment) => {
format!("{}@{} said {}", frame.media, frame.at, segment.said)
}
CaptionIn::Scene(scene) => format!("{} cut", scene.media),
};
emitter.emit(caption);
}
Ok(())
}
}
#[test]
fn a_mixed_node_runs_end_to_end_on_the_thread_engine() {
let world = wired();
let (captions, read) = channel::<ItemId, String>(16);
let mut resident = Resident::new("captioner", Captioner, world.intake).publish(captions);
let Wired {
frames,
segments,
scenes,
..
} = world;
let frame_answers = send(&frames, vec![(0, Frame { media: 7, at: 1 })]);
let scene_answers = send(&scenes, vec![(0, Scene { media: 3 })]);
let segment_answers = send(
&segments,
vec![(
0,
Segment {
media: 7,
said: "hi",
},
)],
);
let orphan = send(&frames, vec![(1, Frame { media: 99, at: 9 })]);
drop(frames);
drop(segments);
drop(scenes);
let engine = std::thread::spawn(move || {
let drained = resident.run();
(drained, resident)
});
let (drained, resident) = engine.join().expect("the engine");
let mut said = Vec::new();
for wake in read.drain() {
let Wake::Delivered(delivery) = wake else {
continue;
};
let (payload, custody) = delivery.into_parts();
let released = payload.keys().copied().collect();
for item in payload {
said.push(item.into_value());
}
if let Some(custody) = custody {
custody.send(crate::Ack::new(
released,
"reader",
crate::Lifecycle::Persisted(()),
));
}
}
said.sort();
assert_eq!(said, ["3 cut", "7@1 said hi"]);
assert_eq!(drained.deliveries(), 2, "one pair and one straight");
for answers in [frame_answers, segment_answers, scene_answers] {
let crate::Answer::Settled(settled) = answers.take() else {
panic!("answered");
};
assert_eq!(settled.released().len(), 1);
}
assert!(
matches!(orphan.take(), crate::Answer::Waiting),
"half a pair is answered by nobody — no timeout, no policy, and the sender's window keeps it"
);
drop(resident);
assert!(
matches!(orphan.take(), crate::Answer::HungUp),
"and when the intake goes, the dropped token is what says held"
);
}
#[test]
fn a_mixed_node_runs_end_to_end_on_the_async_engine() {
let world = wired();
let (captions, read) = channel::<ItemId, String>(16);
let mut resident = Resident::new("captioner", Captioner, world.intake).publish(captions);
let Wired {
frames,
segments,
scenes,
..
} = world;
let _frames = send(&frames, vec![(0, Frame { media: 7, at: 4 })]);
let _segments = send(
&segments,
vec![(
0,
Segment {
media: 7,
said: "there",
},
)],
);
let _scenes = send(&scenes, vec![(0, Scene { media: 8 })]);
drop(frames);
drop(segments);
drop(scenes);
let drained = park_on(resident.run_async(), Duration::from_secs(30)).expect("the loop ended");
assert_eq!(drained.deliveries(), 2);
let mut said = Vec::new();
for wake in read.drain() {
let Wake::Delivered(delivery) = wake else {
continue;
};
for item in delivery.into_items() {
said.push(item.into_value());
}
}
said.sort();
assert_eq!(said, ["7@4 said there", "8 cut"]);
}