chdb_rust/insert_stream.rs
1//! Write-side streaming INSERT.
2//!
3//! The counterpart to [`QueryStream`](crate::query_stream::QueryStream): send
4//! an INSERT statement without data, then push the rows in chunks in any input
5//! format the engine understands. `append` blocks when the engine applies
6//! backpressure, so a producer faster than the engine is throttled rather than
7//! buffered without bound.
8//!
9//! # Ownership
10//!
11//! `chdb_stream_insert_n` returns a handle that is **never null** — an
12//! initialisation failure is reported through `chdb_stream_insert_error`, not
13//! by a null return. The handle must be destroyed on every path, including
14//! error paths, and neither finishing nor cancelling frees it. [`Drop`] handles
15//! the paths [`InsertStream::finish`] and [`InsertStream::cancel`] do not.
16
17use std::ffi::CStr;
18use std::os::raw::c_char;
19use std::time::Duration;
20
21use crate::bindings;
22use crate::connection::Connection;
23use crate::error::{Error, Result};
24use crate::format::InputFormat;
25use crate::query_param::{EncodedParams, QueryParams};
26use crate::query_result::QueryResult;
27
28/// What a finished insert wrote.
29#[derive(Debug, Clone, Copy, PartialEq, Eq)]
30#[non_exhaustive]
31pub struct WriteStats {
32 /// Rows committed by this stream.
33 pub rows_written: u64,
34 /// Bytes committed by this stream.
35 pub bytes_written: u64,
36 /// Wall-clock time the engine spent on the insert.
37 pub elapsed: Duration,
38}
39
40/// An open streaming INSERT.
41///
42/// The connection is exclusively borrowed for the life of the stream: chDB
43/// accepts no other statement on a connection while one is open.
44///
45/// # Examples
46///
47/// ```no_run
48/// use chdb_rust::connection::Connection;
49/// use chdb_rust::format::InputFormat;
50///
51/// let mut conn = Connection::open_in_memory()?;
52/// let mut ins = conn.insert_stream("INSERT INTO t (a, b)", InputFormat::CSV)?;
53/// ins.append(b"1,\"one\"\n")?;
54/// let stats = ins.finish()?;
55/// println!("wrote {} rows", stats.rows_written);
56/// # Ok::<(), chdb_rust::error::Error>(())
57/// ```
58#[derive(Debug)]
59pub struct InsertStream<'a> {
60 handle: bindings::chdb_insert_stream,
61 /// Held so the connection cannot be used while the stream is open.
62 _conn: &'a mut Connection,
63 /// Set by `finish`/`cancel` so `Drop` does not act a second time.
64 done: bool,
65 /// Bytes from `io::Write` calls not yet handed to the engine. Each append
66 /// is a trip across the FFI boundary where the engine copies the chunk,
67 /// so lots of small writes (a `writeln!` per row) are gathered up first.
68 pending: Vec<u8>,
69 /// How much `io::Write` gathers before sending. Zero means no buffering.
70 write_buffer: usize,
71}
72
73/// The default for [`InsertStream::with_write_buffer`].
74pub const DEFAULT_WRITE_BUFFER: usize = 64 * 1024;
75
76impl<'a> InsertStream<'a> {
77 pub(crate) fn start(conn: &'a mut Connection, sql: &str, format: InputFormat) -> Result<Self> {
78 let handle = {
79 let format = format.as_str();
80 // Wraps chdb_stream_insert_n. The handle is never null; an init
81 // failure is reported through chdb_stream_insert_error.
82 unsafe {
83 bindings::chdb_stream_insert_n(
84 conn.handle(),
85 sql.as_ptr() as *const c_char,
86 sql.len(),
87 format.as_ptr() as *const c_char,
88 format.len(),
89 )
90 }
91 };
92
93 let stream = Self {
94 handle,
95 _conn: conn,
96 done: false,
97 pending: Vec::new(),
98 write_buffer: DEFAULT_WRITE_BUFFER,
99 };
100
101 // Checked through the constructed value so that a failed init still
102 // destroys the handle when `stream` is dropped here.
103 stream.check_error()?;
104 Ok(stream)
105 }
106
107 pub(crate) fn start_with_params(
108 conn: &'a mut Connection,
109 sql: &str,
110 format: InputFormat,
111 params: impl Into<QueryParams>,
112 ) -> Result<Self> {
113 let format = format.as_str();
114 let encoded = EncodedParams::encode(params)?;
115
116 let handle = {
117 // Wraps chdb_stream_insert_with_params_n. The handle is never null;
118 // an init failure is reported through chdb_stream_insert_error.
119 // Bindings are captured during initialisation and cleared when it
120 // returns, so `encoded` need only outlive this call. When
121 // `encoded.len() == 0` both pointer args are null, which the C API
122 // accepts for an empty parameter list.
123 unsafe {
124 bindings::chdb_stream_insert_with_params_n(
125 conn.handle(),
126 sql.as_ptr() as *const c_char,
127 sql.len(),
128 format.as_ptr() as *const c_char,
129 format.len(),
130 encoded.names_ptr(),
131 encoded.name_lens_ptr(),
132 encoded.values_ptr(),
133 encoded.value_lens_ptr(),
134 encoded.len(),
135 )
136 }
137 };
138
139 let stream = Self {
140 handle,
141 _conn: conn,
142 done: false,
143 pending: Vec::new(),
144 write_buffer: DEFAULT_WRITE_BUFFER,
145 };
146 stream.check_error()?;
147 Ok(stream)
148 }
149
150 /// The stream's pending error, if it has one.
151 fn check_error(&self) -> Result<()> {
152 // Wraps chdb_stream_insert_error. The returned string is owned by the
153 // stream and is valid until the handle is destroyed.
154 let err = unsafe { bindings::chdb_stream_insert_error(self.handle) };
155 if err.is_null() {
156 return Ok(());
157 }
158
159 let detail = unsafe { CStr::from_ptr(err) }
160 .to_string_lossy()
161 .into_owned();
162 Err(Error::QueryError(detail))
163 }
164
165 /// Set how many bytes `io::Write` gathers before sending them to the engine.
166 ///
167 /// Defaults to [`DEFAULT_WRITE_BUFFER`] (64 KiB). Pass `0` to send every
168 /// write straight away. This only affects writes made through
169 /// [`std::io::Write`]; [`append`](Self::append) always sends immediately.
170 ///
171 /// ```no_run
172 /// use std::io::Write;
173 /// use chdb_rust::connection::Connection;
174 /// use chdb_rust::format::InputFormat;
175 ///
176 /// let mut conn = Connection::open_in_memory()?;
177 /// let mut ins = conn
178 /// .insert_stream("INSERT INTO t (a)", InputFormat::CSV)?
179 /// .with_write_buffer(1024 * 1024);
180 /// writeln!(ins, "1")?;
181 /// ins.finish()?;
182 /// # Ok::<(), Box<dyn std::error::Error>>(())
183 /// ```
184 pub fn with_write_buffer(mut self, bytes: usize) -> Self {
185 self.write_buffer = bytes;
186 self
187 }
188
189 /// Append one chunk of format-encoded data.
190 ///
191 /// The bytes must be in the input format the stream was opened with. Chunk
192 /// boundaries need not align to row boundaries. The data is copied by the
193 /// engine, so the caller keeps ownership.
194 ///
195 /// May block while the engine applies backpressure.
196 ///
197 /// # Errors
198 ///
199 /// Returns [`Error::QueryError`] if the stream has already failed — a
200 /// malformed row in an earlier chunk, for instance — or was finalised.
201 pub fn append(&mut self, data: &[u8]) -> Result<()> {
202 self.flush_pending()?;
203 self.send(data)
204 }
205
206 /// Hands anything gathered by `io::Write` to the engine.
207 fn flush_pending(&mut self) -> Result<()> {
208 if self.pending.is_empty() {
209 return Ok(());
210 }
211 let pending = std::mem::take(&mut self.pending);
212 let sent = self.send(&pending);
213 // Keep the allocation for the next round of writes.
214 self.pending = pending;
215 self.pending.clear();
216 sent
217 }
218
219 fn send(&mut self, data: &[u8]) -> Result<()> {
220 // Wraps chdb_stream_append.
221 let state = unsafe {
222 bindings::chdb_stream_append(
223 self.handle,
224 data.as_ptr() as *const std::ffi::c_void,
225 data.len(),
226 )
227 };
228
229 if state != bindings::chdb_state_CHDBSuccess {
230 // The state says only that it failed; the reason is on the stream.
231 self.check_error()?;
232 return Err(Error::EngineCallFailed {
233 call: "chdb_stream_append",
234 });
235 }
236
237 Ok(())
238 }
239
240 /// Finalise the insert and commit.
241 ///
242 /// # Errors
243 ///
244 /// Returns [`Error::QueryError`] if the engine rejected any of the data.
245 pub fn finish(mut self) -> Result<WriteStats> {
246 // If this fails `done` stays false, so Drop cancels the stream.
247 self.flush_pending()?;
248 self.done = true;
249
250 // Wraps chdb_stream_done. Returns a result that must be destroyed, and
251 // does NOT free the stream handle — Drop still does that.
252 let result_ptr = unsafe { bindings::chdb_stream_done(self.handle) };
253 if result_ptr.is_null() {
254 self.check_error()?;
255 return Err(Error::NoResult);
256 }
257
258 let result = QueryResult::new(result_ptr).check_error()?;
259 Ok(WriteStats {
260 rows_written: result.rows_written(),
261 bytes_written: result.bytes_written(),
262 elapsed: result.elapsed(),
263 })
264 }
265
266 /// Abort without committing.
267 ///
268 /// ClickHouse-default semantics: no rollback of anything already flushed.
269 pub fn cancel(mut self) {
270 self.done = true;
271 // Wraps chdb_stream_cancel_insert. Does not free the handle.
272 unsafe { bindings::chdb_stream_cancel_insert(self.handle) };
273 }
274}
275
276impl Drop for InsertStream<'_> {
277 fn drop(&mut self) {
278 if !self.done {
279 // Wraps chdb_stream_cancel_insert. An unfinished stream is
280 // abandoned, not committed.
281 unsafe { bindings::chdb_stream_cancel_insert(self.handle) };
282 }
283 // Wraps chdb_destroy_insert_stream. Required on every path, finished
284 // or not.
285 unsafe { bindings::chdb_destroy_insert_stream(self.handle) };
286 }
287}
288
289/// Lets anything that writes bytes — `serde_json::to_writer`, `csv::Writer`,
290/// `write!` — feed the stream directly.
291///
292/// Writes are buffered (64 KiB by default, see
293/// [`InsertStream::with_write_buffer`]) before being sent, so a `writeln!` per row
294/// doesn't turn into a trip to the engine per row. Because of that, an error
295/// caused by a write can show up on a later `write`, on `flush`, or on
296/// [`InsertStream::finish`].
297///
298/// Errors are surfaced as
299/// [`std::io::ErrorKind::Other`] carrying the engine's message, because that is
300/// the only shape `io::Write` has for them.
301impl std::io::Write for InsertStream<'_> {
302 fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
303 if self.pending.len() + buf.len() > self.write_buffer {
304 self.flush_pending().map_err(io_err)?;
305 }
306 if buf.len() >= self.write_buffer {
307 // Big enough to be worth sending as it is.
308 self.send(buf).map_err(io_err)?;
309 } else {
310 if self.pending.capacity() == 0 {
311 self.pending.reserve(self.write_buffer);
312 }
313 self.pending.extend_from_slice(buf);
314 }
315 Ok(buf.len())
316 }
317
318 /// Sends anything buffered to the engine. [`InsertStream::finish`] does
319 /// this too, so calling it is only needed to surface errors early.
320 fn flush(&mut self) -> std::io::Result<()> {
321 self.flush_pending().map_err(io_err)
322 }
323}
324
325fn io_err(e: Error) -> std::io::Error {
326 std::io::Error::other(e.to_string())
327}