databricks_zerobus_ingest_sdk/
arrow_configuration.rs

1//! Configuration options for Arrow Flight streams.
2//!
3//! **Beta**: Arrow Flight ingestion is in Beta. The API is stabilising but may
4//! still change before reaching GA.
5
6use crate::stream_options::defaults;
7use arrow_ipc::CompressionType;
8
9/// Configuration options for Arrow Flight stream creation and operation.
10///
11/// These options control the behavior of Arrow Flight ingestion streams, including
12/// backpressure limits, timeout settings, and recovery policies.
13///
14/// **Do not construct this directly.** Configure Arrow streams via the builder API:
15///
16/// ```rust,ignore
17/// let stream = sdk
18///     .stream_builder()
19///     .table("catalog.schema.table")
20///     .oauth("client-id", "client-secret")
21///     .arrow(schema)
22///     .max_inflight_batches(100)
23///     .server_lack_of_ack_timeout_ms(30_000)
24///     .recovery(true)
25///     .build_arrow()
26///     .await?;
27/// ```
28#[derive(Clone, Debug)]
29#[non_exhaustive]
30pub struct ArrowStreamConfigurationOptions {
31    /// Maximum number of batches that can be in-flight (sent but not acknowledged).
32    ///
33    /// This limit controls memory usage and backpressure. When this limit is reached,
34    /// `ingest_batch()` calls will block until acknowledgments free up space.
35    ///
36    /// Default: 1,000
37    pub max_inflight_batches: usize,
38
39    /// Whether to enable automatic stream recovery on failure.
40    ///
41    /// When enabled, the SDK will automatically attempt to reconnect and recover
42    /// the stream when encountering retryable errors.
43    ///
44    /// Default: `true`
45    pub recovery: bool,
46
47    /// Timeout in milliseconds for each stream recovery attempt.
48    ///
49    /// If a recovery attempt takes longer than this, it will be retried.
50    ///
51    /// Default: 15,000 (15 seconds)
52    pub recovery_timeout_ms: u64,
53
54    /// Backoff time in milliseconds between stream recovery retry attempts.
55    ///
56    /// The SDK will wait this duration before attempting another recovery after a failure.
57    ///
58    /// Default: 2,000 (2 seconds)
59    pub recovery_backoff_ms: u64,
60
61    /// Maximum number of recovery retry attempts before giving up.
62    ///
63    /// After this many failed attempts, the stream will close and return an error.
64    ///
65    /// Default: 4
66    pub recovery_retries: u32,
67
68    /// Timeout in milliseconds for waiting for server acknowledgements.
69    ///
70    /// If no acknowledgement is received within this time (and there are pending batches),
71    /// the stream will be considered failed and recovery will be triggered (if enabled).
72    ///
73    /// Default: 60,000 (60 seconds)
74    pub server_lack_of_ack_timeout_ms: u64,
75
76    /// Timeout in milliseconds for flush operations.
77    ///
78    /// If a `flush()` call cannot complete within this time, it will return a timeout error.
79    ///
80    /// Default: 300,000 (5 minutes)
81    pub flush_timeout_ms: u64,
82
83    /// Timeout in milliseconds for stream connection establishment.
84    ///
85    /// If the Arrow Flight stream cannot be established within this time,
86    /// stream creation will fail.
87    ///
88    /// Default: 30,000 (30 seconds)
89    pub connection_timeout_ms: u64,
90
91    /// Optional Arrow IPC compression for Flight payloads.
92    ///
93    /// Supported compression types from `arrow_ipc::CompressionType`:
94    /// - `CompressionType::LZ4_FRAME` - LZ4 frame compression
95    /// - `CompressionType::ZSTD` - Zstandard compression
96    ///
97    /// Default: `None`
98    pub ipc_compression: Option<CompressionType>,
99
100    /// Maximum time in milliseconds to wait during graceful stream close.
101    ///
102    /// When the server sends a close stream signal indicating it will close the stream,
103    /// the SDK enters a "paused" state where it:
104    /// - Continues accepting and buffering new `ingest_batch()` calls
105    /// - Stops sending buffered batches to the server
106    /// - Continues processing acknowledgments for in-flight batches
107    /// - Waits for either all in-flight batches to be acknowledged or the timeout to expire
108    ///
109    /// Configuration values:
110    /// - `None`: Wait for the full server-specified duration (most graceful)
111    /// - `Some(0)`: Immediate recovery, close stream right away
112    /// - `Some(x)`: Wait up to min(x, server_duration) milliseconds
113    ///
114    /// Default: `None` (wait for full server duration)
115    pub stream_paused_max_wait_time_ms: Option<u64>,
116}
117
118impl Default for ArrowStreamConfigurationOptions {
119    fn default() -> Self {
120        Self {
121            max_inflight_batches: 1_000,
122            recovery: defaults::RECOVERY,
123            recovery_timeout_ms: defaults::RECOVERY_TIMEOUT_MS,
124            recovery_backoff_ms: defaults::RECOVERY_BACKOFF_MS,
125            recovery_retries: defaults::RECOVERY_RETRIES,
126            server_lack_of_ack_timeout_ms: defaults::SERVER_LACK_OF_ACK_TIMEOUT_MS,
127            flush_timeout_ms: defaults::FLUSH_TIMEOUT_MS,
128            connection_timeout_ms: defaults::CONNECTION_TIMEOUT_MS,
129            ipc_compression: None,
130            stream_paused_max_wait_time_ms: None,
131        }
132    }
133}
databricks_zerobus_ingest_sdk/arrow_configuration.rs

databricks_zerobus_ingest_sdk/
arrow_configuration.rs
