Skip to main content

chdb_rust/
insert_stream.rs

1//! Write-side streaming INSERT.
2//!
3//! The counterpart to [`QueryStream`](crate::query_stream::QueryStream): send
4//! an INSERT statement without data, then push the rows in chunks in any input
5//! format the engine understands. `append` blocks when the engine applies
6//! backpressure, so a producer faster than the engine is throttled rather than
7//! buffered without bound.
8//!
9//! # Ownership
10//!
11//! `chdb_stream_insert_n` returns a handle that is **never null** — an
12//! initialisation failure is reported through `chdb_stream_insert_error`, not
13//! by a null return. The handle must be destroyed on every path, including
14//! error paths, and neither finishing nor cancelling frees it. [`Drop`] handles
15//! the paths [`InsertStream::finish`] and [`InsertStream::cancel`] do not.
16
17use std::ffi::CStr;
18use std::os::raw::c_char;
19use std::time::Duration;
20
21use crate::bindings;
22use crate::connection::Connection;
23use crate::error::{Error, Result};
24use crate::format::InputFormat;
25use crate::query_param::{EncodedParams, QueryParams};
26use crate::query_result::QueryResult;
27
28/// What a finished insert wrote.
29#[derive(Debug, Clone, Copy, PartialEq, Eq)]
30#[non_exhaustive]
31pub struct WriteStats {
32    /// Rows committed by this stream.
33    pub rows_written: u64,
34    /// Bytes committed by this stream.
35    pub bytes_written: u64,
36    /// Wall-clock time the engine spent on the insert.
37    pub elapsed: Duration,
38}
39
40/// An open streaming INSERT.
41///
42/// The connection is exclusively borrowed for the life of the stream: chDB
43/// accepts no other statement on a connection while one is open.
44///
45/// # Examples
46///
47/// ```no_run
48/// use chdb_rust::connection::Connection;
49/// use chdb_rust::format::InputFormat;
50///
51/// let mut conn = Connection::open_in_memory()?;
52/// let mut ins = conn.insert_stream("INSERT INTO t (a, b)", InputFormat::CSV)?;
53/// ins.append(b"1,\"one\"\n")?;
54/// let stats = ins.finish()?;
55/// println!("wrote {} rows", stats.rows_written);
56/// # Ok::<(), chdb_rust::error::Error>(())
57/// ```
58#[derive(Debug)]
59pub struct InsertStream<'a> {
60    handle: bindings::chdb_insert_stream,
61    /// Held so the connection cannot be used while the stream is open.
62    _conn: &'a mut Connection,
63    /// Set by `finish`/`cancel` so `Drop` does not act a second time.
64    done: bool,
65    /// Bytes from `io::Write` calls not yet handed to the engine. Each append
66    /// is a trip across the FFI boundary where the engine copies the chunk,
67    /// so lots of small writes (a `writeln!` per row) are gathered up first.
68    pending: Vec<u8>,
69    /// How much `io::Write` gathers before sending. Zero means no buffering.
70    write_buffer: usize,
71}
72
73/// The default for [`InsertStream::with_write_buffer`].
74pub const DEFAULT_WRITE_BUFFER: usize = 64 * 1024;
75
76impl<'a> InsertStream<'a> {
77    pub(crate) fn start(conn: &'a mut Connection, sql: &str, format: InputFormat) -> Result<Self> {
78        let handle = {
79            let format = format.as_str();
80            // Wraps chdb_stream_insert_n. The handle is never null; an init
81            // failure is reported through chdb_stream_insert_error.
82            unsafe {
83                bindings::chdb_stream_insert_n(
84                    conn.handle(),
85                    sql.as_ptr() as *const c_char,
86                    sql.len(),
87                    format.as_ptr() as *const c_char,
88                    format.len(),
89                )
90            }
91        };
92
93        let stream = Self {
94            handle,
95            _conn: conn,
96            done: false,
97            pending: Vec::new(),
98            write_buffer: DEFAULT_WRITE_BUFFER,
99        };
100
101        // Checked through the constructed value so that a failed init still
102        // destroys the handle when `stream` is dropped here.
103        stream.check_error()?;
104        Ok(stream)
105    }
106
107    pub(crate) fn start_with_params(
108        conn: &'a mut Connection,
109        sql: &str,
110        format: InputFormat,
111        params: impl Into<QueryParams>,
112    ) -> Result<Self> {
113        let format = format.as_str();
114        let encoded = EncodedParams::encode(params)?;
115
116        let handle = {
117            // Wraps chdb_stream_insert_with_params_n. The handle is never null;
118            // an init failure is reported through chdb_stream_insert_error.
119            // Bindings are captured during initialisation and cleared when it
120            // returns, so `encoded` need only outlive this call. When
121            // `encoded.len() == 0` both pointer args are null, which the C API
122            // accepts for an empty parameter list.
123            unsafe {
124                bindings::chdb_stream_insert_with_params_n(
125                    conn.handle(),
126                    sql.as_ptr() as *const c_char,
127                    sql.len(),
128                    format.as_ptr() as *const c_char,
129                    format.len(),
130                    encoded.names_ptr(),
131                    encoded.name_lens_ptr(),
132                    encoded.values_ptr(),
133                    encoded.value_lens_ptr(),
134                    encoded.len(),
135                )
136            }
137        };
138
139        let stream = Self {
140            handle,
141            _conn: conn,
142            done: false,
143            pending: Vec::new(),
144            write_buffer: DEFAULT_WRITE_BUFFER,
145        };
146        stream.check_error()?;
147        Ok(stream)
148    }
149
150    /// The stream's pending error, if it has one.
151    fn check_error(&self) -> Result<()> {
152        // Wraps chdb_stream_insert_error. The returned string is owned by the
153        // stream and is valid until the handle is destroyed.
154        let err = unsafe { bindings::chdb_stream_insert_error(self.handle) };
155        if err.is_null() {
156            return Ok(());
157        }
158
159        let detail = unsafe { CStr::from_ptr(err) }
160            .to_string_lossy()
161            .into_owned();
162        Err(Error::QueryError(detail))
163    }
164
165    /// Set how many bytes `io::Write` gathers before sending them to the engine.
166    ///
167    /// Defaults to [`DEFAULT_WRITE_BUFFER`] (64 KiB). Pass `0` to send every
168    /// write straight away. This only affects writes made through
169    /// [`std::io::Write`]; [`append`](Self::append) always sends immediately.
170    ///
171    /// ```no_run
172    /// use std::io::Write;
173    /// use chdb_rust::connection::Connection;
174    /// use chdb_rust::format::InputFormat;
175    ///
176    /// let mut conn = Connection::open_in_memory()?;
177    /// let mut ins = conn
178    ///     .insert_stream("INSERT INTO t (a)", InputFormat::CSV)?
179    ///     .with_write_buffer(1024 * 1024);
180    /// writeln!(ins, "1")?;
181    /// ins.finish()?;
182    /// # Ok::<(), Box<dyn std::error::Error>>(())
183    /// ```
184    pub fn with_write_buffer(mut self, bytes: usize) -> Self {
185        self.write_buffer = bytes;
186        self
187    }
188
189    /// Append one chunk of format-encoded data.
190    ///
191    /// The bytes must be in the input format the stream was opened with. Chunk
192    /// boundaries need not align to row boundaries. The data is copied by the
193    /// engine, so the caller keeps ownership.
194    ///
195    /// May block while the engine applies backpressure.
196    ///
197    /// # Errors
198    ///
199    /// Returns [`Error::QueryError`] if the stream has already failed — a
200    /// malformed row in an earlier chunk, for instance — or was finalised.
201    pub fn append(&mut self, data: &[u8]) -> Result<()> {
202        self.flush_pending()?;
203        self.send(data)
204    }
205
206    /// Hands anything gathered by `io::Write` to the engine.
207    fn flush_pending(&mut self) -> Result<()> {
208        if self.pending.is_empty() {
209            return Ok(());
210        }
211        let pending = std::mem::take(&mut self.pending);
212        let sent = self.send(&pending);
213        // Keep the allocation for the next round of writes.
214        self.pending = pending;
215        self.pending.clear();
216        sent
217    }
218
219    fn send(&mut self, data: &[u8]) -> Result<()> {
220        // Wraps chdb_stream_append.
221        let state = unsafe {
222            bindings::chdb_stream_append(
223                self.handle,
224                data.as_ptr() as *const std::ffi::c_void,
225                data.len(),
226            )
227        };
228
229        if state != bindings::chdb_state_CHDBSuccess {
230            // The state says only that it failed; the reason is on the stream.
231            self.check_error()?;
232            return Err(Error::EngineCallFailed {
233                call: "chdb_stream_append",
234            });
235        }
236
237        Ok(())
238    }
239
240    /// Finalise the insert and commit.
241    ///
242    /// # Errors
243    ///
244    /// Returns [`Error::QueryError`] if the engine rejected any of the data.
245    pub fn finish(mut self) -> Result<WriteStats> {
246        // If this fails `done` stays false, so Drop cancels the stream.
247        self.flush_pending()?;
248        self.done = true;
249
250        // Wraps chdb_stream_done. Returns a result that must be destroyed, and
251        // does NOT free the stream handle — Drop still does that.
252        let result_ptr = unsafe { bindings::chdb_stream_done(self.handle) };
253        if result_ptr.is_null() {
254            self.check_error()?;
255            return Err(Error::NoResult);
256        }
257
258        let result = QueryResult::new(result_ptr).check_error()?;
259        Ok(WriteStats {
260            rows_written: result.rows_written(),
261            bytes_written: result.bytes_written(),
262            elapsed: result.elapsed(),
263        })
264    }
265
266    /// Abort without committing.
267    ///
268    /// ClickHouse-default semantics: no rollback of anything already flushed.
269    pub fn cancel(mut self) {
270        self.done = true;
271        // Wraps chdb_stream_cancel_insert. Does not free the handle.
272        unsafe { bindings::chdb_stream_cancel_insert(self.handle) };
273    }
274}
275
276impl Drop for InsertStream<'_> {
277    fn drop(&mut self) {
278        if !self.done {
279            // Wraps chdb_stream_cancel_insert. An unfinished stream is
280            // abandoned, not committed.
281            unsafe { bindings::chdb_stream_cancel_insert(self.handle) };
282        }
283        // Wraps chdb_destroy_insert_stream. Required on every path, finished
284        // or not.
285        unsafe { bindings::chdb_destroy_insert_stream(self.handle) };
286    }
287}
288
289/// Lets anything that writes bytes — `serde_json::to_writer`, `csv::Writer`,
290/// `write!` — feed the stream directly.
291///
292/// Writes are buffered (64 KiB by default, see
293/// [`InsertStream::with_write_buffer`]) before being sent, so a `writeln!` per row
294/// doesn't turn into a trip to the engine per row. Because of that, an error
295/// caused by a write can show up on a later `write`, on `flush`, or on
296/// [`InsertStream::finish`].
297///
298/// Errors are surfaced as
299/// [`std::io::ErrorKind::Other`] carrying the engine's message, because that is
300/// the only shape `io::Write` has for them.
301impl std::io::Write for InsertStream<'_> {
302    fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
303        if self.pending.len() + buf.len() > self.write_buffer {
304            self.flush_pending().map_err(io_err)?;
305        }
306        if buf.len() >= self.write_buffer {
307            // Big enough to be worth sending as it is.
308            self.send(buf).map_err(io_err)?;
309        } else {
310            if self.pending.capacity() == 0 {
311                self.pending.reserve(self.write_buffer);
312            }
313            self.pending.extend_from_slice(buf);
314        }
315        Ok(buf.len())
316    }
317
318    /// Sends anything buffered to the engine. [`InsertStream::finish`] does
319    /// this too, so calling it is only needed to surface errors early.
320    fn flush(&mut self) -> std::io::Result<()> {
321        self.flush_pending().map_err(io_err)
322    }
323}
324
325fn io_err(e: Error) -> std::io::Error {
326    std::io::Error::other(e.to_string())
327}