kafka-client-engine 0.0.1

Execution and driver-integration layer for the native Kafka client
Documentation
//! Effect ordering, timer expiry, release, and terminal publication scenarios.

use std::collections::VecDeque;

use kafka_client_core::{
    BatchExecutionGeneration, BatchExecutionId, BatchId, ByteCount, Deadline, FlushId, Moment,
    OperationId, ProducerBatchPolicy, ProducerEffect, ProducerInput, ProducerMachineError,
    ProducerRetryPolicy,
};

use crate::{
    ProducerDeliveryError, ProducerDeliveryFailureKind, ProducerDeliveryStatus,
    completion::CompletionRegistryError,
};

use super::{
    ProducerHostInvariantError, ProducerHostLimits,
    admission_test::{admit, record},
    cancellation::ProducerRevisionError,
    flush::FlushBindingError,
    host_limits_test::{start, valid_limits},
    terminal_backlog::{ProducerTerminalOwner, RetainedTerminal},
};

#[test]
fn generated_accumulation_waits_until_the_whole_admission_transition_drains() {
    let Ok(batch_policy) = ProducerBatchPolicy::try_new(1, ByteCount::new(u64::MAX), 10) else {
        panic!("test policy should be valid")
    };
    let limits = ProducerHostLimits {
        retained_bytes: 64,
        completion_capacity: 1,
        waiting_record_capacity: 1,
        waiting_byte_capacity: 64,
        record_capacity: 1,
        batch_capacity: 1,
        timer_capacity: 1,
        encoded_byte_capacity: 1_024,
        max_wire_batch_bytes: 1_024,
        max_request_bytes: 1_024,
        max_in_flight_requests_per_broker: 5,
        batch_policy,
        retry_policy: ProducerRetryPolicy::none(),
        compression: kafka_client_core::CompressionPolicy::None,
        compression_worker_count: 0,
        compression_job_capacity: 0,
        compression_byte_capacity: 0,
    };
    let mut host = start(limits);
    let admitted = admit(
        &mut host,
        Moment::from_tick(0),
        Deadline::from_tick(50),
        record("orders"),
    );

    assert_eq!(host.stats().active_timers, 0);
    assert_eq!(host.stats().pending_effects, 1);
    assert!(matches!(
        host.pending_effects(),
        [ProducerEffect::MaterializeBatch { .. }]
    ));
    assert_eq!(host.retry_terminal_backlog(1), Ok(0));
    assert_eq!(host.stats().pending_effects, 1);
    drop(admitted);
}

#[test]
fn due_deadline_releases_engine_bytes_before_publishing_terminal_failure() {
    let mut host = start(valid_limits());
    let admitted = admit(
        &mut host,
        Moment::from_tick(0),
        Deadline::from_tick(5),
        record("orders"),
    );
    let operation_id = admitted.operation_id();
    let observer = admitted.into_delivery_observer();

    assert_eq!(host.fire_due(Moment::from_tick(5), 1), Ok(1));
    let stats = host.stats();
    assert_eq!(stats.store.records, 0);
    assert_eq!(stats.store.bytes, 0);
    assert_eq!(stats.store.batches, 0);
    assert_eq!(stats.core_retained_bytes, ByteCount::new(0));
    assert_eq!(stats.core_completion_slots, 1);
    assert_eq!(stats.active_timers, 0);
    assert_eq!(stats.pending_effects, 0);

    let Err(ProducerDeliveryError::Failed(failure)) = observer.wait() else {
        panic!("deadline should fail the producer operation")
    };
    assert_eq!(failure.kind(), ProducerDeliveryFailureKind::DeadlineElapsed);
    assert_eq!(failure.delivery_status(), ProducerDeliveryStatus::NotSent);
    assert_eq!(operation_id.get(), 1);
}

#[test]
fn notifier_stop_retains_every_same_transition_terminal_in_exact_fifo_order() {
    let mut host = start(valid_limits());
    let first = admit(
        &mut host,
        Moment::from_tick(0),
        Deadline::from_tick(100),
        record("orders"),
    );
    let second = admit(
        &mut host,
        Moment::from_tick(0),
        Deadline::from_tick(100),
        record("orders"),
    );
    let transition = host
        .core
        .apply(ProducerInput::ExecutionUnavailable)
        .unwrap_or_else(|error| panic!("execution stop should plan: {error}"));
    let recovery = host.recover_notifier();

    assert_eq!(
        host.interpret_transition(Moment::from_tick(1), transition),
        Err(ProducerHostInvariantError::Completion(
            CompletionRegistryError::NotifierStopped
        ))
    );
    assert_eq!(host.stats().terminal_backlog, 2);
    assert_eq!(
        host.terminal_front().map(RetainedTerminal::owner),
        Some(ProducerTerminalOwner::Record(first.operation_id()))
    );
    assert_eq!(
        host.terminal_back().map(RetainedTerminal::owner),
        Some(ProducerTerminalOwner::Record(second.operation_id()))
    );
    assert_eq!(host.terminal_publish_attempts(), 1);

    drop((first, second));
    let notifier = recovery
        .notifier
        .unwrap_or_else(|| panic!("notification recovery should remain owned"));
    assert_eq!(notifier.join_off_notifier(), Ok(()));
}

#[test]
fn unbound_flush_terminal_reports_its_typed_failure() {
    let mut host = start(valid_limits());
    let effect = ProducerEffect::CompleteFlush {
        flush_id: FlushId::from_raw(7),
    };

    assert_eq!(
        host.interpret_effect_owned(Moment::from_tick(0), effect),
        Err(ProducerHostInvariantError::FlushBinding(
            FlushBindingError::UnknownFlush
        ))
    );
}

#[test]
fn execution_revision_fails_closed_without_a_cancellation_owner() {
    let mut host = start(valid_limits());
    let generation = BatchExecutionGeneration::try_from_raw(1)
        .unwrap_or_else(|| panic!("test execution generation must be nonzero"));
    let execution = BatchExecutionId::new(BatchId::from_raw(7), generation);
    let effect = ProducerEffect::ReviseBatchExecution {
        previous: execution,
        replacement: None,
        removed_operation_id: OperationId::from_raw(9),
    };

    assert_eq!(
        host.interpret_effect_owned(Moment::from_tick(0), effect),
        Err(ProducerHostInvariantError::Revision(
            ProducerRevisionError::UnexpectedRevisionEffect(execution)
        ))
    );
}

#[test]
fn core_rejection_discards_copied_facts_and_fences_reentry() {
    let mut host = start(valid_limits());
    let current = ProducerInput::RecordAccumulated {
        operation_id: OperationId::from_raw(99),
        batch_id: BatchId::from_raw(88),
        accumulator_bytes: ByteCount::new(7),
        now: Moment::from_tick(3),
    };
    let queued = ProducerInput::ExecutionUnavailable;
    let mut generated = VecDeque::from([current, queued]);

    assert_eq!(
        host.drain_generated(Moment::from_tick(3), &mut generated),
        Err(ProducerHostInvariantError::Core(
            ProducerMachineError::UnknownOperation
        ))
    );
    assert!(generated.is_empty());
    assert_eq!(
        host.poison_reason(),
        Some(ProducerHostInvariantError::Core(
            ProducerMachineError::UnknownOperation
        ))
    );
}

#[test]
fn failing_mechanism_poison_fences_reentry_before_conservative_settlement() {
    let mut host = start(valid_limits());
    let admitted = admit(
        &mut host,
        Moment::from_tick(0),
        Deadline::from_tick(100),
        record("orders"),
    );
    let transition = host
        .core
        .apply(ProducerInput::ExecutionUnavailable)
        .unwrap_or_else(|error| panic!("execution stop should plan: {error}"));
    let repeated = transition.clone();
    host.store.clear_terminal();

    let first = host.interpret_transition(Moment::from_tick(1), transition);
    assert!(matches!(
        first,
        Err(ProducerHostInvariantError::Prepared(_))
    ));
    assert_eq!(host.stats().terminal_backlog, 1);
    assert_eq!(
        host.interpret_transition(Moment::from_tick(2), repeated),
        first
    );

    assert!(host.execution_unavailable(Moment::from_tick(2)).is_err());
    let Err(ProducerDeliveryError::Failed(failure)) = admitted.into_delivery_observer().wait()
    else {
        panic!("the exact planned terminal must settle the accepted operation")
    };
    assert_eq!(
        failure.kind(),
        ProducerDeliveryFailureKind::ExecutionUnavailable
    );
    assert_eq!(failure.delivery_status(), ProducerDeliveryStatus::NotSent);
    assert!(host.terminal_resources_empty());
}