1use chrono::{DateTime, Datelike, TimeZone, Timelike, Utc};
52use futures_core::Stream;
53use pin_project::pin_project;
54use std::collections::{BTreeMap, HashMap, VecDeque};
55use std::hash::Hash;
56use std::marker::PhantomData;
57use std::pin::Pin;
58use std::task::{Context, Poll};
59use std::time::Duration;
60
61pub type Window = (DateTime<Utc>, DateTime<Utc>);
63
64pub trait TimeSeriesData<K> {
66 fn get_key(&self) -> K;
67 fn get_ts(&self) -> DateTime<Utc>;
68}
69
70pub trait Aggregator<Init, Input, Output> {
72 fn init(init: Init) -> Self;
73 fn push(&mut self, item: Input);
74 fn flush(self) -> Output;
75}
76
77pub(crate) fn get_window_start(timestamp: DateTime<Utc>) -> DateTime<Utc> {
80 Utc.with_ymd_and_hms(
81 timestamp.year(),
82 timestamp.month(),
83 timestamp.day(),
84 timestamp.hour(),
85 timestamp.minute(),
86 0,
87 )
88 .unwrap()
89}
90
91#[derive(Clone, Debug)]
97pub struct WindowAggregator<Key, AggInit, AggregatorImpl> {
98 active_windows: HashMap<Key, BTreeMap<DateTime<Utc>, AggregatorImpl>>,
100 current_time: HashMap<Key, DateTime<Utc>>,
103 window_duration: Duration,
105 lateness: Duration,
107 agg_init: AggInit,
108}
109
110impl<Key: Eq + Hash + Clone, AggInit: Clone, AggregatorImpl>
111 WindowAggregator<Key, AggInit, AggregatorImpl>
112{
113 fn new(window_duration: Duration, lateness: Duration, agg_init: AggInit) -> Self {
116 Self {
117 active_windows: HashMap::new(),
118 current_time: HashMap::new(),
119 window_duration,
120 lateness,
121 agg_init,
122 }
123 }
124 fn process_item<Input, AggValue>(
125 &mut self,
126 item: Input,
127 ) -> (
128 impl Iterator<Item = (Window, AggValue)> + '_,
129 impl Iterator<Item = Input> + use<Input, AggValue, Key, AggInit, AggregatorImpl>,
130 )
131 where
132 Input: TimeSeriesData<Key>,
133 AggregatorImpl: Aggregator<AggInit, Input, AggValue>,
134 {
135 let ts = item.get_ts();
136 let key = item.get_key();
137 let mut late_items = Vec::with_capacity(1);
138 let mut split = BTreeMap::new();
139 let current_time = self.current_time.entry(key.clone()).or_insert_with(|| ts);
140 if ts < *current_time - self.lateness {
141 late_items.push(item);
142 } else {
143 *current_time = (*current_time).max(ts);
144 let window_start = get_window_start(ts);
145 let active_windows = self.active_windows.entry(key).or_default();
146 active_windows
149 .entry(window_start)
150 .or_insert_with(|| AggregatorImpl::init(self.agg_init.clone()))
151 .push(item);
152
153 let cutoff_time: DateTime<Utc> =
154 get_window_start((*current_time) - self.lateness) - self.window_duration;
155
156 split = active_windows.split_off(&cutoff_time);
157 std::mem::swap(&mut split, active_windows);
158 if let Some(entry) = active_windows.remove(&cutoff_time) {
159 split.insert(cutoff_time, entry);
160 }
161 }
162 (
163 split.into_iter().map(|(window_start, agg)| {
164 (
165 (window_start, window_start + self.window_duration),
166 agg.flush(),
167 )
168 }),
169 late_items.into_iter(),
170 )
171 }
172
173 fn flush<Input, AggValue>(&mut self) -> impl Iterator<Item = (Window, AggValue)> + '_
176 where
177 AggregatorImpl: Aggregator<AggInit, Input, AggValue>,
178 {
179 let active_windows = std::mem::take(&mut self.active_windows);
180 self.current_time.clear();
181 active_windows
182 .into_values()
183 .flat_map(|x| x.into_iter())
184 .map(move |(ts, agg)| ((ts, ts + self.window_duration), agg.flush()))
185 }
186}
187
188pub struct WindowedAggregationAdaptor<
194 Key,
195 Input,
196 AggInit: Clone,
197 AggValue,
198 AggregatorImpl: Aggregator<AggInit, Input, AggValue>,
199 I: Iterator<Item = Input>,
200> {
201 source: I,
202 aggregator: WindowAggregator<Key, AggInit, AggregatorImpl>,
203 buffer: VecDeque<(Window, AggValue)>,
204 late_buffer: VecDeque<Input>,
205 _agg_fn: AggregatorImpl,
207 _phantom1: PhantomData<Key>,
208 _phantom2: PhantomData<AggInit>,
209}
210
211impl<
212 Key: Eq + Hash + Clone,
213 Input,
214 AggInit: Clone,
215 AggValue,
216 AggregatorImpl: Aggregator<AggInit, Input, AggValue>,
217 I: Iterator<Item = Input>,
218> WindowedAggregationAdaptor<Key, Input, AggInit, AggValue, AggregatorImpl, I>
219{
220 fn new(
221 source: I,
222 duration: Duration,
223 lateness: Duration,
224 agg_init: AggInit,
225 agg_fn: AggregatorImpl,
226 ) -> Self {
227 Self {
228 source,
229 aggregator: WindowAggregator::new(duration, lateness, agg_init),
230 buffer: VecDeque::new(),
231 late_buffer: VecDeque::new(),
232 _agg_fn: agg_fn,
233 _phantom1: PhantomData,
234 _phantom2: PhantomData,
235 }
236 }
237
238 #[inline]
239 fn get_next(&mut self) -> Option<either::Either<(Window, AggValue), Input>> {
240 if let Some(late) = self.late_buffer.pop_front() {
241 return Some(either::Right(late));
242 }
243 if let Some(agg_value) = self.buffer.pop_front() {
244 return Some(either::Left(agg_value));
245 }
246 None
247 }
248}
249
250impl<
251 Key: Eq + Hash + Clone,
252 Input: TimeSeriesData<Key>,
253 AggInit: Clone,
254 AggValue,
255 AggregatorImpl: Aggregator<AggInit, Input, AggValue>,
256 I: Iterator<Item = Input>,
257> Iterator for WindowedAggregationAdaptor<Key, Input, AggInit, AggValue, AggregatorImpl, I>
258{
259 type Item = either::Either<(Window, AggValue), Input>;
260 fn next(&mut self) -> Option<Self::Item> {
261 loop {
262 if let Some(next) = self.get_next() {
264 return Some(next);
265 }
266 match self.source.next() {
268 Some(item) => {
269 let (results, late) = self.aggregator.process_item(item);
271 self.buffer.extend(results);
272 self.late_buffer.extend(late);
273 if let Some(next) = self.get_next() {
275 return Some(next);
276 }
277 }
278 None => {
279 if self.buffer.is_empty() {
281 self.buffer.extend(self.aggregator.flush());
282 }
283 return self.get_next();
284 }
285 }
286 }
287 }
288}
289
290pub trait AggregationWindowingExt<
291 Key: Eq + Hash + Clone,
292 Input: TimeSeriesData<Key>,
293 AggInit: Clone,
294 AggValue,
295 AggregatorImpl: Aggregator<AggInit, Input, AggValue>,
296>: Iterator<Item = Input> + Sized
297{
298 fn window_aggregate(
299 self,
300 window_duration: Duration,
301 lateness: Duration,
302 agg_init: AggInit,
303 agg_fn: AggregatorImpl,
304 ) -> WindowedAggregationAdaptor<Key, Input, AggInit, AggValue, AggregatorImpl, Self> {
305 WindowedAggregationAdaptor::new(self, window_duration, lateness, agg_init, agg_fn)
306 }
307}
308
309impl<
310 Key: Eq + Hash + Clone,
311 Input: TimeSeriesData<Key>,
312 AggInit: Clone,
313 AggValue,
314 AggregatorImpl: Aggregator<AggInit, Input, AggValue>,
315 I: Iterator<Item = Input>,
316> AggregationWindowingExt<Key, Input, AggInit, AggValue, AggregatorImpl> for I
317{
318}
319
320#[pin_project]
321pub struct WindowedAggregationStreamAdaptor<
322 Key,
323 Input,
324 AggInit: Clone,
325 AggValue,
326 AggregatorImpl: Aggregator<AggInit, Input, AggValue>,
327 I: Stream<Item = Input>,
328> {
329 #[pin]
330 source: I,
331 #[pin]
332 aggregator: WindowAggregator<Key, AggInit, AggregatorImpl>,
333 #[pin]
334 buffer: VecDeque<(Window, AggValue)>,
335 #[pin]
336 late_buffer: VecDeque<Input>,
337 _agg_fn: AggregatorImpl,
339 _phantom: PhantomData<(Key, AggInit)>,
340}
341
342impl<
343 Key: Eq + Hash + Clone,
344 Input,
345 AggInit: Clone,
346 AggValue,
347 AggregatorImpl: Aggregator<AggInit, Input, AggValue>,
348 I: Stream<Item = Input>,
349> WindowedAggregationStreamAdaptor<Key, Input, AggInit, AggValue, AggregatorImpl, I>
350{
351 pub fn new(
352 source: I,
353 duration: Duration,
354 lateness: Duration,
355 agg_init: AggInit,
356 agg_fn: AggregatorImpl,
357 ) -> Self {
358 Self {
359 source,
360 aggregator: WindowAggregator::new(duration, lateness, agg_init),
361 buffer: VecDeque::new(),
362 late_buffer: VecDeque::new(),
363 _agg_fn: agg_fn,
364 _phantom: PhantomData,
365 }
366 }
367}
368
369impl<
370 Key: Eq + Hash + Clone + Unpin,
371 Input: TimeSeriesData<Key> + Unpin,
372 AggInit: Clone + Unpin,
373 AggValue: Unpin,
374 AggregatorImpl: Aggregator<AggInit, Input, AggValue>,
375 I: Stream<Item = Input>,
376> Stream for WindowedAggregationStreamAdaptor<Key, Input, AggInit, AggValue, AggregatorImpl, I>
377{
378 type Item = either::Either<(Window, AggValue), Input>;
379 fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
380 let mut this = self.project();
381 loop {
382 if let Some(late) = this.late_buffer.pop_front() {
384 return Poll::Ready(Some(either::Right(late)));
385 }
386 if let Some(agg_value) = this.buffer.pop_front() {
387 return Poll::Ready(Some(either::Left(agg_value)));
388 }
389 match this.source.as_mut().poll_next(cx) {
391 Poll::Ready(Some(item)) => {
392 let (results, late) = this.aggregator.process_item(item);
394 this.buffer.extend(results);
395 this.late_buffer.extend(late);
396 if let Some(late) = this.late_buffer.pop_front() {
398 return Poll::Ready(Some(either::Right(late)));
399 }
400 if let Some(agg_value) = this.buffer.pop_front() {
401 return Poll::Ready(Some(either::Left(agg_value)));
402 }
403 }
404 Poll::Ready(None) => {
405 if this.buffer.is_empty() {
407 this.buffer.extend(this.aggregator.flush());
408 }
409 if let Some(late) = this.late_buffer.pop_front() {
410 return Poll::Ready(Some(either::Right(late)));
411 }
412 if let Some(agg_value) = this.buffer.pop_front() {
413 return Poll::Ready(Some(either::Left(agg_value)));
414 }
415 return Poll::Ready(None);
416 }
417 Poll::Pending => return Poll::Pending,
418 }
419 }
420 }
421}
422
423pub trait AggregationWindowStreamExt<
424 Key: Eq + Hash + Clone + Unpin,
425 Input: TimeSeriesData<Key> + Unpin,
426 AggInit: Clone + Unpin,
427 AggValue,
428 AggregatorImpl: Aggregator<AggInit, Input, AggValue>,
429>: Stream<Item = Input>
430{
431 fn window_aggregate(
432 self,
433 window_duration: Duration,
434 lateness: Duration,
435 agg_init: AggInit,
436 agg_fn: AggregatorImpl,
437 ) -> WindowedAggregationStreamAdaptor<Key, Input, AggInit, AggValue, AggregatorImpl, Self>
438 where
439 Self: Sized,
440 {
441 WindowedAggregationStreamAdaptor::new(self, window_duration, lateness, agg_init, agg_fn)
442 }
443}
444
445impl<
446 Key: Eq + Hash + Clone + Unpin,
447 Input: TimeSeriesData<Key> + Unpin,
448 AggInit: Clone + Unpin,
449 AggValue,
450 AggregatorImpl: Aggregator<AggInit, Input, AggValue>,
451 I: Stream<Item = Input>,
452> AggregationWindowStreamExt<Key, Input, AggInit, AggValue, AggregatorImpl> for I
453{
454}
455
456#[cfg(test)]
457mod tests {
458 use super::*;
459 use chrono::TimeZone;
460 use futures::{StreamExt, stream};
461 use std::ops::Add;
462
463 #[derive(Debug, Clone, PartialEq)]
465 struct TestItem {
466 key: String,
467 ts: DateTime<Utc>,
468 value: i32,
469 }
470
471 impl TimeSeriesData<String> for TestItem {
472 fn get_key(&self) -> String {
473 self.key.clone()
474 }
475 fn get_ts(&self) -> DateTime<Utc> {
476 self.ts
477 }
478 }
479
480 #[derive(Debug, Clone)]
482 struct TestAggregator {
483 sum: i32,
484 }
485
486 impl Aggregator<(), TestItem, i32> for TestAggregator {
487 fn init(_: ()) -> Self {
488 Self { sum: 0 }
489 }
490 fn push(&mut self, item: TestItem) {
491 self.sum += item.value;
492 }
493 fn flush(self) -> i32 {
494 self.sum
495 }
496 }
497
498 #[allow(clippy::type_complexity)]
499 fn get_test_input() -> (Vec<TestItem>, Vec<either::Either<(Window, i32), TestItem>>) {
500 let input = vec![
501 TestItem {
502 key: "key1".to_string(),
503 ts: Utc.with_ymd_and_hms(2025, 1, 1, 0, 0, 0).unwrap(),
504 value: 1,
505 },
506 TestItem {
507 key: "key1".to_string(),
508 ts: Utc.with_ymd_and_hms(2025, 1, 1, 0, 1, 0).unwrap(),
509 value: 3,
510 },
511 TestItem {
513 key: "key1".to_string(),
514 ts: Utc.with_ymd_and_hms(2025, 1, 1, 0, 0, 55).unwrap(),
515 value: 2,
516 },
517 TestItem {
518 key: "key1".to_string(),
519 ts: Utc.with_ymd_and_hms(2025, 1, 1, 0, 1, 30).unwrap(),
520 value: 4,
521 },
522 TestItem {
523 key: "key1".to_string(),
524 ts: Utc.with_ymd_and_hms(2025, 1, 1, 0, 2, 10).unwrap(),
525 value: 5,
526 },
527 TestItem {
529 key: "key1".to_string(),
530 ts: Utc.with_ymd_and_hms(2025, 1, 1, 0, 1, 40).unwrap(),
531 value: 5,
532 },
533 TestItem {
534 key: "key1".to_string(),
535 ts: Utc.with_ymd_and_hms(2025, 1, 1, 0, 3, 10).unwrap(),
536 value: 5,
537 },
538 ];
539 let expected_results_with_late = vec![
540 either::Left((
541 (
542 Utc.with_ymd_and_hms(2025, 1, 1, 0, 0, 0).unwrap(),
543 Utc.with_ymd_and_hms(2025, 1, 1, 0, 1, 0).unwrap(),
544 ),
545 3, )),
547 either::Left((
548 (
549 Utc.with_ymd_and_hms(2025, 1, 1, 0, 1, 0).unwrap(),
550 Utc.with_ymd_and_hms(2025, 1, 1, 0, 2, 0).unwrap(),
551 ),
552 7, )),
554 either::Right(TestItem {
556 key: "key1".to_string(),
557 ts: Utc.with_ymd_and_hms(2025, 1, 1, 0, 1, 40).unwrap(),
558 value: 5,
559 }),
560 either::Left((
561 (
562 Utc.with_ymd_and_hms(2025, 1, 1, 0, 2, 0).unwrap(),
563 Utc.with_ymd_and_hms(2025, 1, 1, 0, 3, 0).unwrap(),
564 ),
565 5, )),
567 either::Left((
568 (
569 Utc.with_ymd_and_hms(2025, 1, 1, 0, 3, 0).unwrap(),
570 Utc.with_ymd_and_hms(2025, 1, 1, 0, 4, 0).unwrap(),
571 ),
572 5, )),
574 ];
575 (input, expected_results_with_late)
576 }
577
578 #[test]
579 fn test_window_aggregator() {
580 let mut window_aggregator = WindowAggregator::<String, (), TestAggregator>::new(
581 Duration::from_secs(60),
582 Duration::from_secs(10),
583 (),
584 );
585 let (items, expected_results) = get_test_input();
586 let expected_on_time: Vec<_> = expected_results[0..expected_results.len() - 1]
589 .iter()
590 .cloned()
591 .flat_map(|i| i.left())
592 .collect();
593 let expected_flush: Vec<_> = expected_results
594 [expected_results.len() - 1..expected_results.len()]
595 .iter()
596 .cloned()
597 .flat_map(|i| i.left())
598 .collect();
599 let expected_late: Vec<_> = expected_results
600 .iter()
601 .cloned()
602 .flat_map(|i| i.right())
603 .collect();
604 let mut results = vec![];
605 let mut late_values = vec![];
606 for item in items {
607 let (processed, late) = window_aggregator.process_item(item);
608 results.extend(processed);
609 late_values.extend(late);
610 }
611 let flushed: Vec<_> = window_aggregator.flush().collect();
612 assert_eq!(results, expected_on_time);
613 assert_eq!(flushed, expected_flush);
614 assert_eq!(late_values, expected_late);
615 }
616
617 #[test]
618 fn test_window_aggregator_iterator() {
619 let (items, expected_results) = get_test_input();
620 let expected_on_time: Vec<_> = expected_results
621 .iter()
622 .cloned()
623 .flat_map(|i| i.left())
624 .collect();
625 let results_with_late: Vec<_> = items
626 .clone()
627 .into_iter()
628 .window_aggregate(
629 Duration::from_secs(60),
630 Duration::from_secs(10),
631 (),
632 TestAggregator::init(()),
633 )
634 .collect();
635 let results_without_late: Vec<_> = items
637 .into_iter()
638 .window_aggregate(
639 Duration::from_secs(60),
640 Duration::from_secs(10),
641 (),
642 TestAggregator::init(()),
643 )
644 .filter_map(|x| x.left())
645 .collect();
646 assert_eq!(results_with_late, expected_results);
647 assert_eq!(results_without_late, expected_on_time);
648 }
649
650 #[tokio::test]
651 async fn test_window_aggregator_stream() {
652 let (items, expected_results) = get_test_input();
653 let expected_on_time: Vec<_> = expected_results
654 .iter()
655 .cloned()
656 .flat_map(|i| i.left())
657 .collect();
658 let results_with_late: Vec<_> = stream::iter(items.clone())
659 .window_aggregate(
660 Duration::from_secs(60),
661 Duration::from_secs(10),
662 (),
663 TestAggregator::init(()),
664 )
665 .collect()
666 .await;
667 let results_without_late: Vec<_> = stream::iter(items)
669 .window_aggregate(
670 Duration::from_secs(60),
671 Duration::from_secs(10),
672 (),
673 TestAggregator::init(()),
674 )
675 .flat_map(move |x| match x {
676 either::Left(l) => stream::iter(vec![l]),
677 either::Right(_) => stream::iter(vec![]),
678 })
679 .collect()
680 .await;
681 assert_eq!(results_with_late, expected_results);
682 assert_eq!(results_without_late, expected_on_time);
683 }
684
685 #[test]
686 fn test_buffer_order() {
687 let base_time = chrono::DateTime::from_timestamp_millis(1738671601000).unwrap();
688 let start = get_window_start(base_time);
689 let items = vec![
690 TestItem {
691 key: "key1".to_string(),
692 ts: base_time,
693 value: 1,
694 },
695 TestItem {
696 key: "key1".to_string(),
697 ts: base_time.add(chrono::Duration::seconds(30)),
698 value: 2,
699 },
700 TestItem {
701 key: "key1".to_string(),
702 ts: base_time.add(chrono::Duration::seconds(30)),
703 value: 3,
704 },
705 TestItem {
706 key: "key1".to_string(),
707 ts: base_time.add(chrono::Duration::minutes(1)),
708 value: 4,
709 },
710 TestItem {
711 key: "key1".to_string(),
712 ts: base_time.add(chrono::Duration::minutes(3)),
713 value: 5,
714 },
715 ];
716 let expected_results = vec![
717 ((start, start.add(chrono::Duration::minutes(1))), 6),
718 (
719 (
720 start.add(chrono::Duration::minutes(1)),
721 start.add(chrono::Duration::minutes(2)),
722 ),
723 4,
724 ),
725 (
726 (
727 start.add(chrono::Duration::minutes(3)),
728 start.add(chrono::Duration::minutes(4)),
729 ),
730 5,
731 ),
732 ];
733 let results_without_late: Vec<_> = items
734 .into_iter()
735 .window_aggregate(
736 Duration::from_secs(60),
737 Duration::from_secs(10),
738 (),
739 TestAggregator::init(()),
740 )
741 .filter_map(|x| x.left())
742 .collect();
743 assert_eq!(results_without_late, expected_results);
744 }
745
746 #[test]
747 fn test_empty_windows() {
748 let items = vec![
749 TestItem {
750 key: "key1".to_string(),
751 ts: Utc.with_ymd_and_hms(2025, 1, 1, 0, 0, 0).unwrap(),
752 value: 1,
753 },
754 TestItem {
755 key: "key1".to_string(),
756 ts: Utc.with_ymd_and_hms(2025, 1, 1, 0, 2, 0).unwrap(),
757 value: 2,
758 },
759 ];
760 let expected_results = vec![
762 (
763 (
764 Utc.with_ymd_and_hms(2025, 1, 1, 0, 0, 0).unwrap(),
765 Utc.with_ymd_and_hms(2025, 1, 1, 0, 1, 0).unwrap(),
766 ),
767 1,
768 ),
769 (
770 (
771 Utc.with_ymd_and_hms(2025, 1, 1, 0, 2, 0).unwrap(),
772 Utc.with_ymd_and_hms(2025, 1, 1, 0, 3, 0).unwrap(),
773 ),
774 2,
775 ),
776 ];
777 let results_without_late: Vec<_> = items
778 .into_iter()
779 .window_aggregate(
780 Duration::from_secs(60),
781 Duration::from_secs(10),
782 (),
783 TestAggregator::init(()),
784 )
785 .filter_map(|x| x.left())
786 .collect();
787 assert_eq!(results_without_late, expected_results);
788 }
789}