Struct DeduplicationOperator 

pub struct DeduplicationOperator { /* private fields */ }

Deduplication operator - filters duplicate events

This operator filters out duplicate events based on a deduplication strategy. It maintains a set of seen event signatures and drops events that match previously processed signatures within the configured TTL window.

Example

use processor::pipeline::operator::DeduplicationOperator;
use processor::config::{DeduplicationConfig, DeduplicationStrategy};
use std::time::Duration;

let config = DeduplicationConfig::new()
    .enabled()
    .with_ttl(Duration::from_secs(3600))
    .with_strategy(DeduplicationStrategy::ContentHash);

let operator = DeduplicationOperator::new("dedup", config);

Implementations

impl DeduplicationOperator

pub fn new<S: Into<String>>(name: S, config: DeduplicationConfig) -> Self

Create a new deduplication operator

pub fn stats(&self) -> DeduplicationStats

Get deduplication statistics

Trait Implementations

impl Clone for DeduplicationOperator

fn clone(&self) -> Self

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for DeduplicationOperator

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl StreamOperator for DeduplicationOperator

fn process<'life0, 'life1, 'async_trait>( &'life0 self, input: Vec<u8>, _ctx: &'life1 OperatorContext, ) -> Pin<Box<dyn Future<Output = Result<Vec<Vec<u8>>>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Process an event and return transformed events

fn name(&self) -> &str

Get the operator name

fn open<'life0, 'async_trait>( &'life0 mut self, ) -> Pin<Box<dyn Future<Output = Result<()>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait,

Called when the operator is initialized

fn close<'life0, 'async_trait>( &'life0 mut self, ) -> Pin<Box<dyn Future<Output = Result<()>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait,

Called when the operator is shut down