use std::{
collections::{BTreeMap, BTreeSet, HashMap, HashSet},
marker::PhantomData,
sync::Arc,
};
use serde::{Deserialize, Serialize};
use thiserror::Error;
use crate::{
ClosedWindow, OpenWindow, TemporalPoint, TemporalRange, WindowBoundaryChange,
WindowBoundaryReason, WindowHistory, WindowRecordId, WindowSegment, WindowTag,
};
type KeySelector<T> = Arc<dyn Fn(&T) -> String + Send + Sync + 'static>;
type ActivePredicate<T> = Arc<dyn Fn(&T) -> bool + Send + Sync + 'static>;
type EventTimeSelector<T> = Arc<dyn Fn(&T) -> i64 + Send + Sync + 'static>;
type SegmentSelector<T> = Arc<dyn Fn(&T) -> Vec<WindowSegment> + Send + Sync + 'static>;
type TagSelector<T> = Arc<dyn Fn(&T) -> Vec<WindowTag> + Send + Sync + 'static>;
type RollupPredicate = Arc<dyn Fn(ChildActivityView) -> bool + Send + Sync + 'static>;
type EmissionCallback = Arc<dyn Fn(&WindowEmission) + Send + Sync + 'static>;
type SegmentTransform =
Arc<dyn Fn(&crate::PrimitiveValue) -> crate::PrimitiveValue + Send + Sync + 'static>;
type RuntimeStateKey = (String, String, Option<String>, Option<String>, String);
type RollupMembershipKey = (String, String, Option<String>, Option<String>, String);
#[derive(Clone, Default)]
struct WindowCallbackSet {
opened: Vec<EmissionCallback>,
closed: Vec<EmissionCallback>,
}
struct RollUpDefinition<T> {
name: String,
key: KeySelector<T>,
is_active: RollupPredicate,
rollups: Vec<RollUpDefinition<T>>,
callbacks: WindowCallbackSet,
segment_projection: RollUpSegmentProjection,
}
struct WindowDefinition<T> {
name: String,
key: KeySelector<T>,
is_active: ActivePredicate<T>,
segments: Option<SegmentSelector<T>>,
tags: Option<TagSelector<T>>,
rollups: Vec<RollUpDefinition<T>>,
callbacks: WindowCallbackSet,
}
impl<T> Clone for RollUpDefinition<T> {
fn clone(&self) -> Self {
Self {
name: self.name.clone(),
key: Arc::clone(&self.key),
is_active: Arc::clone(&self.is_active),
rollups: self.rollups.clone(),
callbacks: self.callbacks.clone(),
segment_projection: self.segment_projection.clone(),
}
}
}
impl<T> Clone for WindowDefinition<T> {
fn clone(&self) -> Self {
Self {
name: self.name.clone(),
key: Arc::clone(&self.key),
is_active: Arc::clone(&self.is_active),
segments: self.segments.as_ref().map(Arc::clone),
tags: self.tags.as_ref().map(Arc::clone),
rollups: self.rollups.clone(),
callbacks: self.callbacks.clone(),
}
}
}
#[derive(Clone, Debug, PartialEq)]
struct OpenState {
id: WindowRecordId,
start: TemporalPoint,
source: Option<String>,
partition: Option<String>,
segments: Vec<WindowSegment>,
tags: Vec<WindowTag>,
}
#[derive(Clone, Debug, Default)]
struct ParentState {
active_children: HashSet<RollupChildId>,
}
#[derive(Clone, Debug, Eq, Hash, PartialEq)]
struct RollupChildId {
key: String,
membership_context: String,
}
#[derive(Clone, Debug, PartialEq)]
struct RollupMembership {
parent_key: String,
segment_context: String,
segments: Vec<WindowSegment>,
tags: Vec<WindowTag>,
}
#[derive(Clone)]
struct ChildContext<'a> {
lineage: &'a str,
key: &'a str,
membership_context: &'a str,
event_point: TemporalPoint,
is_active: bool,
segments: &'a [WindowSegment],
tags: &'a [WindowTag],
}
struct WindowObservation {
state_key: RuntimeStateKey,
window_name: String,
key: String,
event_point: TemporalPoint,
source: Option<String>,
partition: Option<String>,
is_active: bool,
segments: Vec<WindowSegment>,
tags: Vec<WindowTag>,
}
impl ParentState {
fn view(&self) -> ChildActivityView {
ChildActivityView {
active_count: self.active_children.len(),
total_count: self.active_children.len(),
}
}
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub struct ChildActivityView {
pub active_count: usize,
pub total_count: usize,
}
impl ChildActivityView {
#[must_use]
pub fn all_active(self) -> bool {
self.total_count > 0 && self.active_count == self.total_count
}
#[must_use]
pub fn any_active(self) -> bool {
self.active_count > 0
}
}
#[derive(Clone, Default)]
pub struct RollUpSegmentProjection {
preserved_names: Option<BTreeSet<String>>,
dropped_names: BTreeSet<String>,
renamed_names: BTreeMap<String, String>,
value_transforms: BTreeMap<String, SegmentTransform>,
}
impl RollUpSegmentProjection {
#[must_use]
pub fn new() -> Self {
Self::default()
}
#[must_use]
pub fn preserve(mut self, name: impl Into<String>) -> Self {
self.preserved_names
.get_or_insert_with(BTreeSet::new)
.insert(name.into());
self
}
#[must_use]
pub fn drop(mut self, name: impl Into<String>) -> Self {
self.dropped_names.insert(name.into());
self
}
#[must_use]
pub fn rename(mut self, name: impl Into<String>, projected_name: impl Into<String>) -> Self {
self.renamed_names
.insert(name.into(), projected_name.into());
self
}
#[must_use]
pub fn transform<F>(mut self, name: impl Into<String>, transform: F) -> Self
where
F: Fn(&crate::PrimitiveValue) -> crate::PrimitiveValue + Send + Sync + 'static,
{
self.value_transforms
.insert(name.into(), Arc::new(transform));
self
}
}
#[derive(Clone, Default)]
pub struct WindowOptions {
callbacks: WindowCallbackSet,
}
impl WindowOptions {
#[must_use]
pub fn new() -> Self {
Self::default()
}
#[must_use]
pub fn on_opened<F>(mut self, callback: F) -> Self
where
F: Fn(&WindowEmission) + Send + Sync + 'static,
{
self.callbacks.opened.push(Arc::new(callback));
self
}
#[must_use]
pub fn on_closed<F>(mut self, callback: F) -> Self
where
F: Fn(&WindowEmission) + Send + Sync + 'static,
{
self.callbacks.closed.push(Arc::new(callback));
self
}
}
#[derive(Clone, Debug, Eq, Error, PartialEq)]
pub enum EventPipelineBuildError {
#[error("window name cannot be empty")]
EmptyWindowName,
#[error("duplicate window name '{0}'")]
DuplicateWindowName(String),
#[error("invalid segment projection: {0}")]
InvalidSegmentProjection(String),
}
#[derive(Clone, Debug, Eq, Error, PartialEq)]
pub enum IngestionError {
#[error(transparent)]
Temporal(#[from] crate::TemporalRangeError),
#[error("processing position overflow")]
ProcessingPositionOverflow,
#[error("window record id overflow")]
RecordIdOverflow,
#[error("invalid segment projection: {0}")]
InvalidSegmentProjection(String),
}
#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)]
pub enum WindowTransitionKind {
Opened,
Closed,
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct WindowEmission {
pub kind: WindowTransitionKind,
pub window_name: String,
pub key: String,
pub record_id: WindowRecordId,
pub position: i64,
pub source: Option<String>,
pub partition: Option<String>,
pub segments: Vec<WindowSegment>,
pub tags: Vec<WindowTag>,
pub boundary_reason: Option<WindowBoundaryReason>,
pub boundary_changes: Vec<WindowBoundaryChange>,
}
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
pub struct IngestionResult {
pub processing_position: i64,
pub emissions: Vec<WindowEmission>,
}
impl IngestionResult {
#[must_use]
pub fn has_emissions(&self) -> bool {
!self.emissions.is_empty()
}
}
#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
pub struct WindowMetadata {
pub name: String,
pub rollups: Vec<WindowMetadata>,
}
#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
pub struct EventPipelineMetadata {
pub event_type: Option<String>,
pub windows: Vec<WindowMetadata>,
}
#[derive(Clone)]
pub struct EventPipelineBuilder<T> {
windows: Vec<WindowDefinition<T>>,
event_time: Option<EventTimeSelector<T>>,
emission_callbacks: Vec<EmissionCallback>,
record_windows: bool,
marker: PhantomData<T>,
}
impl<T> Default for EventPipelineBuilder<T> {
fn default() -> Self {
Self {
windows: Vec::new(),
event_time: None,
emission_callbacks: Vec::new(),
record_windows: false,
marker: PhantomData,
}
}
}
pub struct WindowPipelineBuilder<T> {
builder: EventPipelineBuilder<T>,
path: Vec<usize>,
}
pub struct EventPipeline<T> {
windows: Vec<WindowDefinition<T>>,
event_time: Option<EventTimeSelector<T>>,
emission_callbacks: Vec<EmissionCallback>,
window_callbacks: BTreeMap<String, WindowCallbackSet>,
record_windows: bool,
history: WindowHistory,
active: HashMap<RuntimeStateKey, OpenState>,
parents: HashMap<RuntimeStateKey, ParentState>,
rollup_memberships: HashMap<RollupMembershipKey, RollupMembership>,
position: i64,
next_record_id: u64,
marker: PhantomData<T>,
}
#[must_use]
pub fn for_events<T>() -> EventPipelineBuilder<T> {
EventPipelineBuilder::default()
}
impl<T> EventPipelineBuilder<T> {
#[must_use]
pub fn record_windows(mut self) -> Self {
self.record_windows = true;
self
}
#[must_use]
pub fn with_event_time<F>(mut self, selector: F) -> Self
where
F: Fn(&T) -> i64 + Send + Sync + 'static,
{
self.event_time = Some(Arc::new(selector));
self
}
#[must_use]
pub fn on_emission<F>(mut self, callback: F) -> Self
where
F: Fn(&WindowEmission) + Send + Sync + 'static,
{
self.emission_callbacks.push(Arc::new(callback));
self
}
#[must_use]
pub fn track_window<K, F, P>(mut self, name: impl Into<String>, key: F, is_active: P) -> Self
where
K: Into<String> + 'static,
F: Fn(&T) -> K + Send + Sync + 'static,
P: Fn(&T) -> bool + Send + Sync + 'static,
{
self.windows.push(WindowDefinition {
name: name.into(),
key: Arc::new(move |event| key(event).into()),
is_active: Arc::new(is_active),
segments: None,
tags: None,
rollups: Vec::new(),
callbacks: WindowCallbackSet::default(),
});
self
}
#[must_use]
pub fn track_window_with_options<K, F, P, C>(
mut self,
name: impl Into<String>,
key: F,
is_active: P,
configure: C,
) -> Self
where
K: Into<String> + 'static,
F: Fn(&T) -> K + Send + Sync + 'static,
P: Fn(&T) -> bool + Send + Sync + 'static,
C: FnOnce(WindowOptions) -> WindowOptions,
{
let options = configure(WindowOptions::new());
self.windows.push(WindowDefinition {
name: name.into(),
key: Arc::new(move |event| key(event).into()),
is_active: Arc::new(is_active),
segments: None,
tags: None,
rollups: Vec::new(),
callbacks: options.callbacks,
});
self
}
#[must_use]
pub fn track_window_with_metadata<K, F, P, S, G>(
mut self,
name: impl Into<String>,
key: F,
is_active: P,
segments: S,
tags: G,
) -> Self
where
K: Into<String> + 'static,
F: Fn(&T) -> K + Send + Sync + 'static,
P: Fn(&T) -> bool + Send + Sync + 'static,
S: Fn(&T) -> Vec<WindowSegment> + Send + Sync + 'static,
G: Fn(&T) -> Vec<WindowTag> + Send + Sync + 'static,
{
self.windows.push(WindowDefinition {
name: name.into(),
key: Arc::new(move |event| key(event).into()),
is_active: Arc::new(is_active),
segments: Some(Arc::new(segments)),
tags: Some(Arc::new(tags)),
rollups: Vec::new(),
callbacks: WindowCallbackSet::default(),
});
self
}
#[must_use]
pub fn track_window_with_metadata_and_options<K, F, P, S, G, C>(
mut self,
name: impl Into<String>,
key: F,
is_active: P,
segments: S,
tags: G,
configure: C,
) -> Self
where
K: Into<String> + 'static,
F: Fn(&T) -> K + Send + Sync + 'static,
P: Fn(&T) -> bool + Send + Sync + 'static,
S: Fn(&T) -> Vec<WindowSegment> + Send + Sync + 'static,
G: Fn(&T) -> Vec<WindowTag> + Send + Sync + 'static,
C: FnOnce(WindowOptions) -> WindowOptions,
{
let options = configure(WindowOptions::new());
self.windows.push(WindowDefinition {
name: name.into(),
key: Arc::new(move |event| key(event).into()),
is_active: Arc::new(is_active),
segments: Some(Arc::new(segments)),
tags: Some(Arc::new(tags)),
rollups: Vec::new(),
callbacks: options.callbacks,
});
self
}
#[must_use]
pub fn window<K, F, P>(
mut self,
name: impl Into<String>,
key: F,
is_active: P,
) -> WindowPipelineBuilder<T>
where
K: Into<String> + 'static,
F: Fn(&T) -> K + Send + Sync + 'static,
P: Fn(&T) -> bool + Send + Sync + 'static,
{
self.windows.push(WindowDefinition {
name: name.into(),
key: Arc::new(move |event| key(event).into()),
is_active: Arc::new(is_active),
segments: None,
tags: None,
rollups: Vec::new(),
callbacks: WindowCallbackSet::default(),
});
WindowPipelineBuilder {
path: vec![self.windows.len() - 1],
builder: self,
}
}
#[must_use]
pub fn window_with_options<K, F, P, C>(
mut self,
name: impl Into<String>,
key: F,
is_active: P,
configure: C,
) -> WindowPipelineBuilder<T>
where
K: Into<String> + 'static,
F: Fn(&T) -> K + Send + Sync + 'static,
P: Fn(&T) -> bool + Send + Sync + 'static,
C: FnOnce(WindowOptions) -> WindowOptions,
{
let options = configure(WindowOptions::new());
self.windows.push(WindowDefinition {
name: name.into(),
key: Arc::new(move |event| key(event).into()),
is_active: Arc::new(is_active),
segments: None,
tags: None,
rollups: Vec::new(),
callbacks: options.callbacks,
});
WindowPipelineBuilder {
path: vec![self.windows.len() - 1],
builder: self,
}
}
#[must_use]
pub fn window_with_metadata<K, F, P, S, G>(
mut self,
name: impl Into<String>,
key: F,
is_active: P,
segments: S,
tags: G,
) -> WindowPipelineBuilder<T>
where
K: Into<String> + 'static,
F: Fn(&T) -> K + Send + Sync + 'static,
P: Fn(&T) -> bool + Send + Sync + 'static,
S: Fn(&T) -> Vec<WindowSegment> + Send + Sync + 'static,
G: Fn(&T) -> Vec<WindowTag> + Send + Sync + 'static,
{
self.windows.push(WindowDefinition {
name: name.into(),
key: Arc::new(move |event| key(event).into()),
is_active: Arc::new(is_active),
segments: Some(Arc::new(segments)),
tags: Some(Arc::new(tags)),
rollups: Vec::new(),
callbacks: WindowCallbackSet::default(),
});
WindowPipelineBuilder {
path: vec![self.windows.len() - 1],
builder: self,
}
}
#[must_use]
pub fn window_with_metadata_and_options<K, F, P, S, G, C>(
mut self,
name: impl Into<String>,
key: F,
is_active: P,
segments: S,
tags: G,
configure: C,
) -> WindowPipelineBuilder<T>
where
K: Into<String> + 'static,
F: Fn(&T) -> K + Send + Sync + 'static,
P: Fn(&T) -> bool + Send + Sync + 'static,
S: Fn(&T) -> Vec<WindowSegment> + Send + Sync + 'static,
G: Fn(&T) -> Vec<WindowTag> + Send + Sync + 'static,
C: FnOnce(WindowOptions) -> WindowOptions,
{
let options = configure(WindowOptions::new());
self.windows.push(WindowDefinition {
name: name.into(),
key: Arc::new(move |event| key(event).into()),
is_active: Arc::new(is_active),
segments: Some(Arc::new(segments)),
tags: Some(Arc::new(tags)),
rollups: Vec::new(),
callbacks: options.callbacks,
});
WindowPipelineBuilder {
path: vec![self.windows.len() - 1],
builder: self,
}
}
pub fn build(self) -> Result<EventPipeline<T>, EventPipelineBuildError> {
self.try_build()
}
#[must_use]
#[track_caller]
pub fn build_or_panic(self) -> EventPipeline<T> {
self.try_build().expect("valid pipeline configuration")
}
pub fn try_build(self) -> Result<EventPipeline<T>, EventPipelineBuildError> {
validate_window_names(&self.windows)?;
validate_segment_projections(&self.windows)?;
let window_callbacks = collect_window_callbacks(&self.windows);
Ok(EventPipeline {
windows: self.windows,
event_time: self.event_time,
emission_callbacks: self.emission_callbacks,
window_callbacks,
record_windows: self.record_windows,
history: WindowHistory::new(),
active: HashMap::new(),
parents: HashMap::new(),
rollup_memberships: HashMap::new(),
position: 0,
next_record_id: 0,
marker: PhantomData,
})
}
}
impl<T> WindowPipelineBuilder<T> {
#[must_use]
pub fn on_emission<F>(mut self, callback: F) -> Self
where
F: Fn(&WindowEmission) + Send + Sync + 'static,
{
self.builder.emission_callbacks.push(Arc::new(callback));
self
}
#[must_use]
pub fn roll_up<K, F, P>(mut self, name: impl Into<String>, key: F, is_active: P) -> Self
where
K: Into<String> + 'static,
F: Fn(&T) -> K + Send + Sync + 'static,
P: Fn(ChildActivityView) -> bool + Send + Sync + 'static,
{
let definition = RollUpDefinition {
name: name.into(),
key: Arc::new(move |event| key(event).into()),
is_active: Arc::new(is_active),
rollups: Vec::new(),
callbacks: WindowCallbackSet::default(),
segment_projection: RollUpSegmentProjection::default(),
};
let next_index = add_rollup(&mut self.builder.windows, &self.path, definition);
self.path.push(next_index);
self
}
#[must_use]
pub fn roll_up_with_options<K, F, P, C>(
mut self,
name: impl Into<String>,
key: F,
is_active: P,
configure: C,
) -> Self
where
K: Into<String> + 'static,
F: Fn(&T) -> K + Send + Sync + 'static,
P: Fn(ChildActivityView) -> bool + Send + Sync + 'static,
C: FnOnce(WindowOptions) -> WindowOptions,
{
let options = configure(WindowOptions::new());
let definition = RollUpDefinition {
name: name.into(),
key: Arc::new(move |event| key(event).into()),
is_active: Arc::new(is_active),
rollups: Vec::new(),
callbacks: options.callbacks,
segment_projection: RollUpSegmentProjection::default(),
};
let next_index = add_rollup(&mut self.builder.windows, &self.path, definition);
self.path.push(next_index);
self
}
#[must_use]
pub fn roll_up_with_segment_projection<K, F, P, C>(
mut self,
name: impl Into<String>,
key: F,
is_active: P,
configure_projection: C,
) -> Self
where
K: Into<String> + 'static,
F: Fn(&T) -> K + Send + Sync + 'static,
P: Fn(ChildActivityView) -> bool + Send + Sync + 'static,
C: FnOnce(RollUpSegmentProjection) -> RollUpSegmentProjection,
{
let definition = RollUpDefinition {
name: name.into(),
key: Arc::new(move |event| key(event).into()),
is_active: Arc::new(is_active),
rollups: Vec::new(),
callbacks: WindowCallbackSet::default(),
segment_projection: configure_projection(RollUpSegmentProjection::new()),
};
let next_index = add_rollup(&mut self.builder.windows, &self.path, definition);
self.path.push(next_index);
self
}
#[must_use]
pub fn roll_up_with_segment_projection_and_options<K, F, P, C, O>(
mut self,
name: impl Into<String>,
key: F,
is_active: P,
configure_projection: C,
configure_options: O,
) -> Self
where
K: Into<String> + 'static,
F: Fn(&T) -> K + Send + Sync + 'static,
P: Fn(ChildActivityView) -> bool + Send + Sync + 'static,
C: FnOnce(RollUpSegmentProjection) -> RollUpSegmentProjection,
O: FnOnce(WindowOptions) -> WindowOptions,
{
let options = configure_options(WindowOptions::new());
let definition = RollUpDefinition {
name: name.into(),
key: Arc::new(move |event| key(event).into()),
is_active: Arc::new(is_active),
rollups: Vec::new(),
callbacks: options.callbacks,
segment_projection: configure_projection(RollUpSegmentProjection::new()),
};
let next_index = add_rollup(&mut self.builder.windows, &self.path, definition);
self.path.push(next_index);
self
}
pub fn build(self) -> Result<EventPipeline<T>, EventPipelineBuildError> {
self.builder.build()
}
#[must_use]
#[track_caller]
pub fn build_or_panic(self) -> EventPipeline<T> {
self.builder.build_or_panic()
}
pub fn try_build(self) -> Result<EventPipeline<T>, EventPipelineBuildError> {
self.builder.try_build()
}
}
impl<T> EventPipeline<T> {
#[must_use]
pub fn processing_position(&self) -> i64 {
self.position
}
#[must_use]
pub fn history(&self) -> &WindowHistory {
&self.history
}
#[must_use]
pub fn metadata(&self) -> EventPipelineMetadata {
EventPipelineMetadata {
event_type: Some(std::any::type_name::<T>().to_owned()),
windows: self.windows.iter().map(window_metadata).collect(),
}
}
pub fn ingest(
&mut self,
event: T,
source: Option<&str>,
partition: Option<&str>,
) -> Result<IngestionResult, IngestionError> {
let next_position = self
.position
.checked_add(1)
.ok_or(IngestionError::ProcessingPositionOverflow)?;
let max_new_records = self
.windows
.iter()
.map(window_definition_count)
.try_fold(0_u64, |total, count| total.checked_add(count))
.ok_or(IngestionError::RecordIdOverflow)?;
if self.next_record_id > u64::MAX.saturating_sub(max_new_records) {
return Err(IngestionError::RecordIdOverflow);
}
let event_point = self.event_time.as_ref().map_or_else(
|| TemporalPoint::position(next_position),
|selector| TemporalPoint::timestamp_ticks(selector(&event)),
);
for active in self.active.values() {
TemporalRange::new(active.start.clone(), event_point.clone())?;
}
self.preflight_segment_projections(&event)?;
self.position = next_position;
let mut emissions = Vec::new();
let windows = std::mem::take(&mut self.windows);
let mut ingestion_error = None;
for definition in &windows {
if let Err(error) = self.ingest_definition(
definition,
&event,
event_point.clone(),
source,
partition,
&mut emissions,
) {
ingestion_error = Some(error);
break;
}
}
self.windows = windows;
if let Some(error) = ingestion_error {
return Err(error);
}
for emission in &emissions {
self.invoke_callbacks(emission);
}
Ok(IngestionResult {
processing_position: self.position,
emissions,
})
}
fn preflight_segment_projections(&self, event: &T) -> Result<(), IngestionError> {
for definition in &self.windows {
let segments = if (definition.is_active)(event) {
definition
.segments
.as_ref()
.map_or_else(Vec::new, |selector| selector(event))
} else {
Vec::new()
};
preflight_rollup_projections(&definition.rollups, &segments)?;
}
Ok(())
}
pub fn ingest_many<I>(
&mut self,
events: I,
source: Option<&str>,
partition: Option<&str>,
) -> Result<IngestionResult, IngestionError>
where
I: IntoIterator<Item = T>,
{
let mut emissions = Vec::new();
for event in events {
emissions.extend(self.ingest(event, source, partition)?.emissions);
}
Ok(IngestionResult {
processing_position: self.position,
emissions,
})
}
fn ingest_definition(
&mut self,
definition: &WindowDefinition<T>,
event: &T,
event_point: TemporalPoint,
source: Option<&str>,
partition: Option<&str>,
emissions: &mut Vec<WindowEmission>,
) -> Result<(), IngestionError> {
let key = (definition.key)(event);
let is_active = (definition.is_active)(event);
let segments = if is_active {
definition
.segments
.as_ref()
.map_or_else(Vec::new, |selector| selector(event))
} else {
Vec::new()
};
let tags = if is_active {
definition
.tags
.as_ref()
.map_or_else(Vec::new, |selector| selector(event))
} else {
Vec::new()
};
let state_key = (
definition.name.clone(),
key.clone(),
source.map(str::to_owned),
partition.map(str::to_owned),
String::new(),
);
let previous = self.active.get(&state_key).cloned();
emissions.extend(self.sync_window_state(WindowObservation {
state_key,
window_name: definition.name.clone(),
key: key.clone(),
event_point: event_point.clone(),
source: source.map(str::to_owned),
partition: partition.map(str::to_owned),
is_active,
segments: segments.clone(),
tags: tags.clone(),
})?);
if is_active {
self.sync_rollups(
&definition.rollups,
event,
source,
partition,
ChildContext {
lineage: &definition.name,
key: &key,
membership_context: "",
event_point: event_point.clone(),
is_active: true,
segments: &segments,
tags: &tags,
},
emissions,
)?;
} else if let Some(previous) = previous {
self.sync_rollups(
&definition.rollups,
event,
source,
partition,
ChildContext {
lineage: &definition.name,
key: &key,
membership_context: "",
event_point: event_point.clone(),
is_active: false,
segments: &previous.segments,
tags: &previous.tags,
},
emissions,
)?;
} else {
self.sync_rollups(
&definition.rollups,
event,
source,
partition,
ChildContext {
lineage: &definition.name,
key: &key,
membership_context: "",
event_point: event_point.clone(),
is_active: false,
segments: &[],
tags: &[],
},
emissions,
)?;
}
Ok(())
}
fn sync_rollups(
&mut self,
definitions: &[RollUpDefinition<T>],
event: &T,
source: Option<&str>,
partition: Option<&str>,
child: ChildContext<'_>,
emissions: &mut Vec<WindowEmission>,
) -> Result<(), IngestionError> {
for definition in definitions {
self.sync_rollup(
definition,
event,
source,
partition,
child.clone(),
emissions,
)?;
}
Ok(())
}
fn sync_rollup(
&mut self,
definition: &RollUpDefinition<T>,
event: &T,
source: Option<&str>,
partition: Option<&str>,
child: ChildContext<'_>,
emissions: &mut Vec<WindowEmission>,
) -> Result<(), IngestionError> {
let projected_segments = project_segments(&definition.segment_projection, child.segments)?;
let segment_context = stable_segments(&projected_segments);
let parent_key = (definition.key)(event);
let rollup_lineage = format!("{}>{}", child.lineage, definition.name);
let membership_key = (
rollup_lineage.clone(),
child.key.to_owned(),
source.map(str::to_owned),
partition.map(str::to_owned),
child.membership_context.to_owned(),
);
let current_membership = RollupMembership {
parent_key,
segment_context,
segments: projected_segments,
tags: child.tags.to_vec(),
};
let previous_membership = self.rollup_memberships.get(&membership_key).cloned();
if child.is_active {
if let Some(previous) = previous_membership.as_ref().filter(|previous| {
previous.parent_key != current_membership.parent_key
|| previous.segment_context != current_membership.segment_context
}) {
self.update_rollup_parent(
definition,
event,
source,
partition,
&rollup_lineage,
child.key,
child.membership_context,
child.event_point.clone(),
previous,
false,
emissions,
)?;
}
self.rollup_memberships
.insert(membership_key, current_membership.clone());
self.update_rollup_parent(
definition,
event,
source,
partition,
&rollup_lineage,
child.key,
child.membership_context,
child.event_point,
¤t_membership,
true,
emissions,
)?;
} else if let Some(previous) = previous_membership {
self.rollup_memberships.remove(&membership_key);
self.update_rollup_parent(
definition,
event,
source,
partition,
&rollup_lineage,
child.key,
child.membership_context,
child.event_point,
&previous,
false,
emissions,
)?;
}
Ok(())
}
#[allow(clippy::too_many_arguments)]
fn update_rollup_parent(
&mut self,
definition: &RollUpDefinition<T>,
event: &T,
source: Option<&str>,
partition: Option<&str>,
rollup_lineage: &str,
child_key: &str,
child_membership_context: &str,
event_point: TemporalPoint,
membership: &RollupMembership,
child_is_active: bool,
emissions: &mut Vec<WindowEmission>,
) -> Result<(), IngestionError> {
let parent_state_key = (
rollup_lineage.to_owned(),
membership.parent_key.clone(),
source.map(str::to_owned),
partition.map(str::to_owned),
membership.segment_context.clone(),
);
let (is_active, is_empty) = {
let parent_state = self.parents.entry(parent_state_key.clone()).or_default();
let child_id = RollupChildId {
key: child_key.to_owned(),
membership_context: child_membership_context.to_owned(),
};
if child_is_active {
parent_state.active_children.insert(child_id);
} else {
parent_state.active_children.remove(&child_id);
}
(
(definition.is_active)(parent_state.view()),
parent_state.active_children.is_empty(),
)
};
if is_empty && !is_active {
self.parents.remove(&parent_state_key);
}
emissions.extend(self.sync_window_state(WindowObservation {
state_key: (
definition.name.clone(),
membership.parent_key.clone(),
source.map(str::to_owned),
partition.map(str::to_owned),
membership.segment_context.clone(),
),
window_name: definition.name.clone(),
key: membership.parent_key.clone(),
event_point: event_point.clone(),
source: source.map(str::to_owned),
partition: partition.map(str::to_owned),
is_active,
segments: membership.segments.clone(),
tags: membership.tags.clone(),
})?);
self.sync_rollups(
&definition.rollups,
event,
source,
partition,
ChildContext {
lineage: &definition.name,
key: &membership.parent_key,
membership_context: &membership.segment_context,
event_point,
is_active,
segments: &membership.segments,
tags: &membership.tags,
},
emissions,
)?;
Ok(())
}
fn sync_window_state(
&mut self,
observation: WindowObservation,
) -> Result<Vec<WindowEmission>, IngestionError> {
if observation.is_active {
if let Some(previous) = self.active.get(&observation.state_key) {
if previous.segments == observation.segments {
if previous.tags != observation.tags {
if let Some(state) = self.active.get_mut(&observation.state_key) {
state.tags = observation.tags.clone();
}
if self.record_windows
&& let Some(id) = self
.active
.get(&observation.state_key)
.map(|state| state.id.clone())
{
self.history.update_open_tags(&id, observation.tags.clone());
}
}
return Ok(Vec::new());
}
let mut emissions = Vec::new();
let changes = segment_changes(&previous.segments, &observation.segments);
if let Some(emission) = self.close_window_state(
&observation.state_key,
observation.event_point.clone(),
Some(WindowBoundaryReason::SegmentChanged),
changes,
)? {
emissions.push(emission);
}
emissions.push(self.open_window_state(observation)?);
return Ok(emissions);
}
return Ok(vec![self.open_window_state(observation)?]);
}
Ok(self
.close_window_state(
&observation.state_key,
observation.event_point,
Some(WindowBoundaryReason::ActivePredicateEnded),
Vec::new(),
)?
.into_iter()
.collect())
}
fn open_window_state(
&mut self,
observation: WindowObservation,
) -> Result<WindowEmission, IngestionError> {
let id = self.next_id()?;
let open = OpenWindow {
id: id.clone(),
window_name: observation.window_name.clone(),
key: observation.key.clone(),
start: observation.event_point.clone(),
known_at: None,
source: observation.source.clone(),
partition: observation.partition.clone(),
segments: observation.segments.clone(),
tags: observation.tags.clone(),
};
if self.record_windows {
self.history.push_open(open);
}
self.active.insert(
observation.state_key,
OpenState {
id: id.clone(),
start: observation.event_point.clone(),
source: observation.source.clone(),
partition: observation.partition.clone(),
segments: observation.segments.clone(),
tags: observation.tags.clone(),
},
);
Ok(WindowEmission {
kind: WindowTransitionKind::Opened,
window_name: observation.window_name,
key: observation.key,
record_id: id,
position: self.position,
source: observation.source,
partition: observation.partition,
segments: observation.segments,
tags: observation.tags,
boundary_reason: None,
boundary_changes: Vec::new(),
})
}
fn close_window_state(
&mut self,
state_key: &RuntimeStateKey,
event_point: TemporalPoint,
boundary_reason: Option<WindowBoundaryReason>,
boundary_changes: Vec<WindowBoundaryChange>,
) -> Result<Option<WindowEmission>, IngestionError> {
let Some(open_state) = self.active.get(state_key).cloned() else {
return Ok(None);
};
let range = TemporalRange::new(open_state.start.clone(), event_point.clone())?;
self.active.remove(state_key);
let emission = WindowEmission {
kind: WindowTransitionKind::Closed,
window_name: state_key.0.clone(),
key: state_key.1.clone(),
record_id: open_state.id.clone(),
position: self.position,
source: open_state.source.clone(),
partition: open_state.partition.clone(),
segments: open_state.segments.clone(),
tags: open_state.tags.clone(),
boundary_reason,
boundary_changes: boundary_changes.clone(),
};
if self.record_windows {
self.history.remove_open(&open_state.id);
self.history.push_closed(ClosedWindow {
id: open_state.id,
window_name: state_key.0.clone(),
key: state_key.1.clone(),
range,
known_at: None,
source: open_state.source,
partition: open_state.partition,
segments: open_state.segments,
tags: open_state.tags,
boundary_reason,
boundary_changes,
});
}
Ok(Some(emission))
}
fn invoke_callbacks(&self, emission: &WindowEmission) {
if let Some(callbacks) = self.window_callbacks.get(&emission.window_name) {
let selected = if emission.kind == WindowTransitionKind::Opened {
&callbacks.opened
} else {
&callbacks.closed
};
for callback in selected {
callback(emission);
}
}
for callback in &self.emission_callbacks {
callback(emission);
}
}
fn next_id(&mut self) -> Result<WindowRecordId, IngestionError> {
let id = WindowRecordId::generated(format!("pipeline-{:04}", self.next_record_id));
self.next_record_id = self
.next_record_id
.checked_add(1)
.ok_or(IngestionError::RecordIdOverflow)?;
Ok(id)
}
}
fn window_metadata<T>(definition: &WindowDefinition<T>) -> WindowMetadata {
WindowMetadata {
name: definition.name.clone(),
rollups: definition.rollups.iter().map(rollup_metadata).collect(),
}
}
fn window_definition_count<T>(definition: &WindowDefinition<T>) -> u64 {
1 + definition
.rollups
.iter()
.map(rollup_definition_count)
.sum::<u64>()
}
fn rollup_definition_count<T>(definition: &RollUpDefinition<T>) -> u64 {
1 + definition
.rollups
.iter()
.map(rollup_definition_count)
.sum::<u64>()
}
fn rollup_metadata<T>(definition: &RollUpDefinition<T>) -> WindowMetadata {
WindowMetadata {
name: definition.name.clone(),
rollups: definition.rollups.iter().map(rollup_metadata).collect(),
}
}
fn validate_window_names<T>(
windows: &[WindowDefinition<T>],
) -> Result<(), EventPipelineBuildError> {
let mut names = BTreeSet::new();
for window in windows {
validate_window_name(&window.name, &mut names)?;
for rollup in &window.rollups {
validate_rollup_name(rollup, &mut names)?;
}
}
Ok(())
}
fn validate_segment_projections<T>(
windows: &[WindowDefinition<T>],
) -> Result<(), EventPipelineBuildError> {
for window in windows {
for rollup in &window.rollups {
validate_rollup_projection(rollup)?;
}
}
Ok(())
}
fn validate_rollup_projection<T>(
rollup: &RollUpDefinition<T>,
) -> Result<(), EventPipelineBuildError> {
let mut projected_names = BTreeSet::new();
for (original, projected) in &rollup.segment_projection.renamed_names {
if original.trim().is_empty() || projected.trim().is_empty() {
return Err(EventPipelineBuildError::InvalidSegmentProjection(
"segment names cannot be empty".to_owned(),
));
}
if !projected_names.insert(projected) {
return Err(EventPipelineBuildError::InvalidSegmentProjection(format!(
"multiple renames target '{projected}'"
)));
}
if rollup
.segment_projection
.renamed_names
.keys()
.any(|name| name != original && name == projected)
{
return Err(EventPipelineBuildError::InvalidSegmentProjection(format!(
"rename target '{projected}' collides with a source name"
)));
}
}
for child in &rollup.rollups {
validate_rollup_projection(child)?;
}
Ok(())
}
fn validate_window_name(
name: &str,
names: &mut BTreeSet<String>,
) -> Result<(), EventPipelineBuildError> {
if name.trim().is_empty() {
return Err(EventPipelineBuildError::EmptyWindowName);
}
if !names.insert(name.to_owned()) {
return Err(EventPipelineBuildError::DuplicateWindowName(
name.to_owned(),
));
}
Ok(())
}
fn validate_rollup_name<T>(
rollup: &RollUpDefinition<T>,
names: &mut BTreeSet<String>,
) -> Result<(), EventPipelineBuildError> {
validate_window_name(&rollup.name, names)?;
for child in &rollup.rollups {
validate_rollup_name(child, names)?;
}
Ok(())
}
fn collect_window_callbacks<T>(
windows: &[WindowDefinition<T>],
) -> BTreeMap<String, WindowCallbackSet> {
let mut callbacks = BTreeMap::new();
for window in windows {
collect_window_callbacks_for_window(window, &mut callbacks);
}
callbacks
}
fn collect_window_callbacks_for_window<T>(
window: &WindowDefinition<T>,
callbacks: &mut BTreeMap<String, WindowCallbackSet>,
) {
callbacks.insert(window.name.clone(), window.callbacks.clone());
for rollup in &window.rollups {
collect_window_callbacks_for_rollup(rollup, callbacks);
}
}
fn collect_window_callbacks_for_rollup<T>(
rollup: &RollUpDefinition<T>,
callbacks: &mut BTreeMap<String, WindowCallbackSet>,
) {
callbacks.insert(rollup.name.clone(), rollup.callbacks.clone());
for child in &rollup.rollups {
collect_window_callbacks_for_rollup(child, callbacks);
}
}
fn project_segments(
projection: &RollUpSegmentProjection,
segments: &[WindowSegment],
) -> Result<Vec<WindowSegment>, IngestionError> {
if segments.is_empty()
|| (projection.preserved_names.is_none()
&& projection.dropped_names.is_empty()
&& projection.renamed_names.is_empty()
&& projection.value_transforms.is_empty())
{
return Ok(segments.to_vec());
}
let mut projected = Vec::new();
let mut selected_original_names = BTreeSet::new();
let mut selected_projected_names = BTreeSet::new();
for segment in segments {
if !should_keep_segment(projection, &segment.name) {
continue;
}
let projected_name = projection
.renamed_names
.get(&segment.name)
.cloned()
.unwrap_or_else(|| segment.name.clone());
if !selected_projected_names.insert(projected_name.clone()) {
return Err(IngestionError::InvalidSegmentProjection(format!(
"projected segment '{projected_name}' is not unique"
)));
}
let value = projection.value_transforms.get(&segment.name).map_or_else(
|| segment.value.clone(),
|transform| transform(&segment.value),
);
projected.push(WindowSegment {
name: projected_name,
value,
parent_name: segment.parent_name.clone(),
});
selected_original_names.insert(segment.name.clone());
}
for segment in &mut projected {
if let Some(parent_name) = &segment.parent_name {
if selected_original_names.contains(parent_name) {
segment.parent_name = projection
.renamed_names
.get(parent_name)
.cloned()
.or_else(|| Some(parent_name.clone()));
} else {
segment.parent_name = None;
}
}
}
Ok(projected)
}
fn preflight_rollup_projections<T>(
definitions: &[RollUpDefinition<T>],
child_segments: &[WindowSegment],
) -> Result<(), IngestionError> {
for definition in definitions {
let projected = project_segments(&definition.segment_projection, child_segments)?;
preflight_rollup_projections(&definition.rollups, &projected)?;
}
Ok(())
}
fn should_keep_segment(projection: &RollUpSegmentProjection, name: &str) -> bool {
if projection.dropped_names.contains(name) {
return false;
}
projection
.preserved_names
.as_ref()
.is_none_or(|names| names.contains(name))
}
fn stable_segments(segments: &[WindowSegment]) -> String {
let mut stable = String::new();
for segment in segments {
let parent = segment.parent_name.as_deref().unwrap_or_default();
let value = serde_json::to_string(&segment.value).unwrap_or_default();
for part in [parent, segment.name.as_str(), value.as_str()] {
stable.push_str(&part.len().to_string());
stable.push(':');
stable.push_str(part);
stable.push('|');
}
}
stable
}
fn segment_changes(
previous: &[WindowSegment],
current: &[WindowSegment],
) -> Vec<WindowBoundaryChange> {
let mut changes = Vec::new();
let mut names = BTreeSet::new();
names.extend(previous.iter().map(|segment| segment.name.as_str()));
names.extend(current.iter().map(|segment| segment.name.as_str()));
for name in names {
let before = previous.iter().find(|segment| segment.name == name);
let after = current.iter().find(|segment| segment.name == name);
if before == after {
continue;
}
changes.push(WindowBoundaryChange {
segment_name: name.to_owned(),
previous_value: before.map(|segment| segment.value.clone()),
current_value: after.map(|segment| segment.value.clone()),
});
}
changes
}
fn add_rollup<T>(
windows: &mut [WindowDefinition<T>],
path: &[usize],
definition: RollUpDefinition<T>,
) -> usize {
let mut rollups = &mut windows[path[0]].rollups;
for index in &path[1..] {
rollups = &mut rollups[*index].rollups;
}
rollups.push(definition);
rollups.len() - 1
}
#[cfg(test)]
#[path = "pipeline_tests.rs"]
mod tests;