use std::collections::{BTreeMap, BTreeSet};
use std::sync::Arc;
use crate::batching::main::compress_chunks::compress_chunks;
use crate::batching::models::{Batch, BatchRow, CompressedChunk, Partition};
use crate::config::models::OversizedPolicy;
use crate::delivery::constants::CONFLICT;
use crate::delivery::errors::DeliveryError;
use crate::delivery::models::{Confirmation, DeliveryReport, StreamingDelivery};
use crate::http::errors::ApiError;
use crate::naming::models::Stream;
use crate::streaming_api::main::append_rows::append_rows;
use crate::streaming_api::main::channel_statuses::channel_statuses;
use crate::streaming_api::main::open_channel::open_channel;
use crate::streaming_api::models::{AppendRequest, ChannelStatus, StreamingClient};
struct Chunked {
chunks: Vec<CompressedChunk>,
skipped: u64,
}
impl StreamingDelivery {
pub(crate) fn new(
client: Arc<StreamingClient>,
stream: Stream,
max_request_bytes: u64,
oversized: OversizedPolicy,
) -> Self {
Self {
client,
stream,
max_request_bytes,
oversized,
continuations: BTreeMap::new(),
pending: BTreeMap::new(),
error_rows: BTreeMap::new(),
}
}
fn channel(&self, partition: &Partition) -> String {
self.stream.channel(&partition.topic, partition.partition)
}
pub(crate) async fn open(
&mut self,
partition: &Partition,
) -> Result<Option<i64>, DeliveryError> {
let channel = self.channel(partition);
let opened = open_channel(&self.client, &self.stream.streaming_pipe, &channel)
.await
.map_err(open_error)?;
self.continuations
.insert(partition.clone(), opened.continuation_token);
self.error_rows.insert(channel, opened.error_rows);
self.pending.remove(partition);
Ok(opened.committed_offset)
}
pub(crate) fn is_open(&self, partition: &Partition) -> bool {
self.continuations.contains_key(partition)
}
pub(crate) async fn deliver(&mut self, batch: &Batch) -> Result<DeliveryReport, DeliveryError> {
let mut report = DeliveryReport::default();
for (partition, rows) in batch.by_partition() {
if !self.is_open(&partition) {
return Err(DeliveryError::Reset(format!(
"channel for {}/{} is not open",
partition.topic, partition.partition
)));
}
let chunked = self.chunk(rows)?;
report.skipped_rows += chunked.skipped;
for chunk in chunked.chunks {
report = add_chunk(report, &chunk);
self.append(&partition, chunk).await?;
}
}
Ok(report)
}
async fn append(
&mut self,
partition: &Partition,
chunk: CompressedChunk,
) -> Result<(), DeliveryError> {
let continuation = self
.continuations
.get(partition)
.cloned()
.unwrap_or_default();
let channel = self.channel(partition);
let request = AppendRequest {
pipe: &self.stream.streaming_pipe,
channel: &channel,
continuation: &continuation,
offset: chunk.last_offset,
body: chunk.body,
};
let next = append_rows(&self.client, request)
.await
.map_err(|error| reset("append", &error))?;
self.continuations.insert(partition.clone(), next);
self.pending.insert(partition.clone(), chunk.last_offset);
Ok(())
}
fn chunk(&self, mut rows: Vec<&BatchRow>) -> Result<Chunked, DeliveryError> {
let mut skipped = 0;
loop {
match compress_chunks(&rows, self.max_request_bytes) {
Ok(chunks) => return Ok(Chunked { chunks, skipped }),
Err(oversized) if self.oversized == OversizedPolicy::Skip => {
tracing::warn!(
topic = %oversized.topic,
partition = oversized.partition,
offset = oversized.offset,
compressed_bytes = oversized.compressed_bytes,
"skipping a row too large for one streaming request"
);
rows.retain(|row| row.offset != oversized.offset);
skipped += 1;
}
Err(oversized) => {
return Err(DeliveryError::Fatal(format!(
"row {}/{}@{} is {} bytes after gzip, over the {}-byte request limit; set oversized = \"skip\" to drop such rows",
oversized.topic,
oversized.partition,
oversized.offset,
oversized.compressed_bytes,
self.max_request_bytes
)));
}
}
}
}
pub(crate) fn has_pending(&self) -> bool {
!self.pending.is_empty()
}
pub(crate) async fn poll(&mut self) -> Result<Confirmation, DeliveryError> {
if self.pending.is_empty() {
return Ok(Confirmation::default());
}
let names: BTreeSet<String> = self
.pending
.keys()
.map(|partition| self.channel(partition))
.collect();
let names: Vec<String> = names.into_iter().collect();
let statuses = channel_statuses(&self.client, &self.stream.streaming_pipe, &names)
.await
.map_err(|error| reset("channel status", &error))?;
let pending = std::mem::take(&mut self.pending);
let mut confirmation = Confirmation::default();
for (partition, offset) in pending {
let channel = self.channel(&partition);
let status = statuses.get(&channel).ok_or_else(|| {
DeliveryError::Reset(format!(
"Snowflake reported no status for channel {channel}"
))
})?;
confirmation = self.settle(confirmation, partition, offset, status)?;
}
Ok(confirmation)
}
fn settle(
&mut self,
mut confirmation: Confirmation,
partition: Partition,
offset: i64,
status: &ChannelStatus,
) -> Result<Confirmation, DeliveryError> {
let channel = self.channel(&partition);
if !status.is_healthy() {
return Err(DeliveryError::Reset(format!(
"channel {channel} reports {}: {}",
status.channel_status_code.clone().unwrap_or_default(),
status.last_error_message.clone().unwrap_or_default()
)));
}
confirmation.error_rows +=
self.error_delta(&channel, status.rows_error_count.unwrap_or_default());
let committed = status.committed_offset();
if let Some(committed) = committed {
confirmation
.confirmed
.insert(partition.clone(), committed.min(offset));
}
if committed.is_none_or(|committed| committed < offset) {
self.pending.insert(partition, offset);
}
Ok(confirmation)
}
fn error_delta(&mut self, channel: &str, total: i64) -> u64 {
let previous = self
.error_rows
.insert(channel.to_owned(), total)
.unwrap_or_default();
total.saturating_sub(previous).max(0) as u64
}
pub(crate) fn reset(&mut self) {
self.continuations.clear();
self.pending.clear();
}
pub(crate) fn forget(&mut self, partitions: &[Partition]) {
for partition in partitions {
self.continuations.remove(partition);
self.pending.remove(partition);
}
}
}
fn add_chunk(mut report: DeliveryReport, chunk: &CompressedChunk) -> DeliveryReport {
report.rows += chunk.rows as u64;
report.uncompressed_bytes += chunk.uncompressed_bytes;
report.sent_bytes += chunk.body.len() as u64;
report.requests += 1;
report
}
fn reset(action: &str, error: &ApiError) -> DeliveryError {
DeliveryError::Reset(format!("{action} failed: {error}"))
}
fn open_error(error: ApiError) -> DeliveryError {
match error.is_retryable() || error.status() == Some(CONFLICT) {
true => DeliveryError::Retry(format!("opening the channel failed: {error}")),
false => DeliveryError::Fatal(format!("opening the channel failed: {error}")),
}
}