bigquery 0.10.0

Library provides a simple API for Google BigQuery based on the gRPC Storage Read/Write and v2 APIs
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
//! Rows grouped into append requests that never exceed the request cap.
//!
//! BigQuery accepted an append of 20,488,556 bytes and rejected one of 20,989,006 with a bare
//! `InvalidArgument` that ends the whole connection, so a batch is sized exactly before it is
//! sent and never split after a rejection.

use crate::errors::BigQueryCodecErrorKind;
use crate::types::error::CodecError;
use crate::write::arrow::ArrowWriterSchema;
use crate::write::descriptor::WritePlan;
use crate::BigQueryMissingValue;
use gcloud_sdk::google::cloud::bigquery::storage::v1::append_rows_request::{
    self, ArrowData, MissingValueInterpretation, ProtoData,
};
use gcloud_sdk::google::cloud::bigquery::storage::v1::{
    AppendRowsRequest, ArrowRecordBatch, ArrowSchema, ProtoRows, ProtoSchema,
};
use gcloud_sdk::prost::encoding::encoded_len_varint;
use gcloud_sdk::prost::Message;
use std::sync::Arc;
use tokio::time::Instant;

/// The largest `max_request_bytes` a writer accepts.
pub(crate) const MAX_REQUEST_BYTES: usize = 19_000_000;

/// The two length prefixes around the rows, of `ProtoData` and `ProtoRows` or of `ArrowData`
/// and `ArrowRecordBatch`, take one byte each when empty and at most four below 2^28 bytes,
/// which is far above the cap.
const NESTED_LENGTH_SLACK: usize = 2 * 3;

/// What every request of one writer carries besides its rows and offset.
#[derive(Debug, Clone)]
pub(crate) struct RequestTarget {
    pub(crate) write_stream: String,
    pub(crate) trace_id: Option<String>,
    pub(crate) missing_value: Option<BigQueryMissingValue>,
}

/// Builds one append request for `rows`, with their writer schema when `with_schema`; a
/// connection needs it on its first request and after the schema changed.
pub(crate) fn append_request(
    target: &RequestTarget,
    offset: Option<i64>,
    rows: &BatchRows,
    with_schema: bool,
) -> AppendRowsRequest {
    let rows = match rows {
        BatchRows::Proto { plan, rows } => append_rows_request::Rows::ProtoRows(ProtoData {
            writer_schema: with_schema.then(|| ProtoSchema {
                proto_descriptor: Some(plan.descriptor().clone()),
            }),
            rows: Some(ProtoRows {
                serialized_rows: rows.clone(),
            }),
        }),
        BatchRows::Arrow {
            schema,
            record_batch,
            ..
        } => append_rows_request::Rows::ArrowRows(ArrowData {
            writer_schema: with_schema.then(|| ArrowSchema {
                serialized_schema: schema.serialized().to_vec(),
            }),
            rows: Some(ArrowRecordBatch {
                serialized_record_batch: record_batch.clone(),
                ..Default::default()
            }),
        }),
    };
    AppendRowsRequest {
        // Every request names the stream: after an in-band schema switch BigQuery treats the
        // connection as multiplexed and rejects requests without it.
        write_stream: target.write_stream.clone(),
        offset,
        trace_id: target.trace_id.clone().unwrap_or_default(),
        default_missing_value_interpretation: match target.missing_value {
            None => MissingValueInterpretation::Unspecified,
            Some(BigQueryMissingValue::Null) => MissingValueInterpretation::NullValue,
            Some(BigQueryMissingValue::DefaultValue) => MissingValueInterpretation::DefaultValue,
        }
        .into(),
        rows: Some(rows),
        ..Default::default()
    }
}

/// The writer schema rows were encoded against.
#[derive(Debug, Clone)]
pub(crate) enum WriterSchema {
    Proto(Arc<WritePlan>),
    Arrow(Arc<ArrowWriterSchema>),
}

impl WriterSchema {
    /// Whether both are the same schema value, which is how a connection tells it already sent
    /// the schema of a batch.
    pub(crate) fn is(&self, other: &WriterSchema) -> bool {
        match (self, other) {
            (WriterSchema::Proto(one), WriterSchema::Proto(other)) => Arc::ptr_eq(one, other),
            (WriterSchema::Arrow(one), WriterSchema::Arrow(other)) => Arc::ptr_eq(one, other),
            _ => false,
        }
    }

    /// The bytes of a request that are not rows, with the writer schema always counted: a batch
    /// can become the first request of a new connection when it is resent after a reconnect.
    fn request_overhead(&self, target: &RequestTarget) -> usize {
        let empty = match self {
            WriterSchema::Proto(plan) => BatchRows::Proto {
                plan: plan.clone(),
                rows: Vec::new(),
            },
            WriterSchema::Arrow(schema) => BatchRows::Arrow {
                schema: schema.clone(),
                record_batch: Vec::new(),
                row_count: 0,
            },
        };
        append_request(target, Some(i64::MAX), &empty, true).encoded_len() + NESTED_LENGTH_SLACK
    }
}

/// The rows of a batch, encoded against their writer schema.
#[derive(Debug)]
pub(crate) enum BatchRows {
    /// Rows encoded one by one as protobuf messages.
    Proto {
        plan: Arc<WritePlan>,
        rows: Vec<Vec<u8>>,
    },
    /// One IPC record batch message.
    Arrow {
        schema: Arc<ArrowWriterSchema>,
        record_batch: Vec<u8>,
        row_count: u64,
    },
}

impl BatchRows {
    pub(crate) fn row_count(&self) -> u64 {
        match self {
            BatchRows::Proto { rows, .. } => rows.len() as u64,
            BatchRows::Arrow { row_count, .. } => *row_count,
        }
    }

    pub(crate) fn schema(&self) -> WriterSchema {
        match self {
            BatchRows::Proto { plan, .. } => WriterSchema::Proto(plan.clone()),
            BatchRows::Arrow { schema, .. } => WriterSchema::Arrow(schema.clone()),
        }
    }
}

/// A sealed batch: rows encoded against one writer schema, the unit of sending, acknowledging
/// and resending.
#[derive(Debug)]
pub(crate) struct Batch {
    pub(crate) index: u64,
    pub(crate) first_row: u64,
    pub(crate) rows: BatchRows,
    /// The bytes the rows take in the request, framing included.
    pub(crate) bytes: usize,
    /// How many times it was sent.
    pub(crate) attempts: usize,
}

impl Batch {
    pub(crate) fn row_count(&self) -> u64 {
        self.rows.row_count()
    }
}

#[derive(Debug)]
struct OpenBatch {
    first_row: u64,
    rows: Vec<Vec<u8>>,
    bytes: usize,
    since: Instant,
}

/// Groups encoded rows into batches that fit one request under `max_request_bytes`.
#[derive(Debug)]
pub(crate) struct Batcher {
    plan: Arc<WritePlan>,
    target: RequestTarget,
    max_request_bytes: usize,
    max_rows: Option<usize>,
    overhead: usize,
    open: Option<OpenBatch>,
    next_index: u64,
    next_row: u64,
}

impl Batcher {
    pub(crate) fn new(
        plan: Arc<WritePlan>,
        target: &RequestTarget,
        max_request_bytes: usize,
        max_rows: Option<usize>,
    ) -> Self {
        let overhead = WriterSchema::Proto(plan.clone()).request_overhead(target);
        Batcher {
            plan,
            target: target.clone(),
            max_request_bytes,
            max_rows,
            overhead,
            open: None,
            next_index: 0,
            next_row: 0,
        }
    }

    /// The bytes a request reserves for everything but its rows.
    #[cfg(test)]
    pub(crate) fn overhead(&self) -> usize {
        self.overhead
    }

    /// The bytes left for rows in one request.
    pub(crate) fn capacity(&self) -> usize {
        self.max_request_bytes.saturating_sub(self.overhead)
    }

    /// The bytes left for an Arrow record batch in one request with `schema`.
    pub(crate) fn arrow_capacity(&self, schema: &Arc<ArrowWriterSchema>) -> usize {
        let overhead = WriterSchema::Arrow(schema.clone()).request_overhead(&self.target);
        self.max_request_bytes.saturating_sub(overhead)
    }

    /// The most rows of one batch, `max_batch_rows`.
    pub(crate) fn max_rows(&self) -> Option<usize> {
        self.max_rows
    }

    /// The write-order index the next row gets.
    pub(crate) fn next_row(&self) -> u64 {
        self.next_row
    }

    /// What a row of `len` encoded bytes takes in a request: its tag, its length and itself.
    ///
    /// # Errors
    /// `RowTooLarge` when the row cannot fit any request alone.
    pub(crate) fn cost(&self, len: usize) -> Result<usize, CodecError> {
        let cost = 1 + encoded_len_varint(len as u64) + len;
        if cost > self.capacity() {
            return Err(CodecError::new(
                BigQueryCodecErrorKind::RowTooLarge,
                format!(
                    "the row takes {cost} bytes and a request has room for {} rows bytes under \
                     max_request_bytes = {}",
                    self.capacity(),
                    self.max_request_bytes
                ),
            ));
        }
        Ok(cost)
    }

    /// Whether the open batch, if any, can take one more row of `cost`.
    pub(crate) fn fits(&self, cost: usize) -> bool {
        match &self.open {
            None => true,
            Some(open) => {
                open.bytes + cost <= self.capacity()
                    && self.max_rows.is_none_or(|max| open.rows.len() < max)
            }
        }
    }

    /// Whether the open batch has reached `max_batch_rows`.
    pub(crate) fn is_full(&self) -> bool {
        match (&self.open, self.max_rows) {
            (Some(open), Some(max)) => open.rows.len() >= max,
            _ => false,
        }
    }

    /// Adds a row to the open batch, opening one at `now` if there is none. Returns whether
    /// it opened a batch. The caller checked [`fits`](Self::fits) first.
    pub(crate) fn push(&mut self, row: Vec<u8>, cost: usize, now: Instant) -> bool {
        let opened = self.open.is_none();
        let next_row = self.next_row;
        let open = self.open.get_or_insert_with(|| OpenBatch {
            first_row: next_row,
            rows: Vec::new(),
            bytes: 0,
            since: now,
        });
        open.rows.push(row);
        open.bytes += cost;
        self.next_row += 1;
        opened
    }

    /// When the open batch got its first row.
    pub(crate) fn open_since(&self) -> Option<Instant> {
        self.open.as_ref().map(|open| open.since)
    }

    /// The bytes of the open batch's rows.
    pub(crate) fn open_bytes(&self) -> Option<usize> {
        self.open.as_ref().map(|open| open.bytes)
    }

    /// Closes the open batch, if any.
    pub(crate) fn seal(&mut self) -> Option<Batch> {
        let open = self.open.take()?;
        let index = self.next_index;
        self.next_index += 1;
        Some(Batch {
            index,
            first_row: open.first_row,
            rows: BatchRows::Proto {
                plan: self.plan.clone(),
                rows: open.rows,
            },
            bytes: open.bytes,
            attempts: 0,
        })
    }

    /// Makes `rows` a batch of their own, taking `bytes` in the request. There must be no open
    /// batch, so that batches stay in write order.
    pub(crate) fn seal_rows(&mut self, rows: BatchRows, bytes: usize) -> Batch {
        debug_assert!(self.open.is_none(), "batches stay in write order");
        let index = self.next_index;
        self.next_index += 1;
        let first_row = self.next_row;
        self.next_row += rows.row_count();
        Batch {
            index,
            first_row,
            rows,
            bytes,
            attempts: 0,
        }
    }

    /// The plan for the rows from now on. The open batch must be sealed first, since a batch
    /// holds rows of one plan only.
    pub(crate) fn set_plan(&mut self, plan: Arc<WritePlan>) {
        debug_assert!(self.open.is_none(), "a batch holds rows of one plan only");
        self.overhead = WriterSchema::Proto(plan.clone()).request_overhead(&self.target);
        self.plan = plan;
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::types::testkit::field;
    use crate::{BigQueryFieldMode, BigQueryFieldType, BigQueryTableSchema};
    use gcloud_sdk::prost::Message;
    use proptest::prelude::*;
    use std::sync::Arc;

    const STREAM: &str =
        "projects/p/datasets/shop/tables/orders/streams/Cic2NjQ2ZjZkNjE2OTZlEhAKCgoKCgoKCgoKCgoK";

    fn plan(columns: usize) -> Arc<WritePlan> {
        let fields = (0..columns)
            .map(|i| {
                field(
                    &format!("column_with_a_long_name_{i}"),
                    BigQueryFieldType::String { max_length: None },
                    BigQueryFieldMode::Nullable,
                )
            })
            .collect();
        Arc::new(WritePlan::new(&BigQueryTableSchema { fields }, false))
    }

    fn target() -> RequestTarget {
        RequestTarget {
            write_stream: STREAM.to_string(),
            trace_id: Some("bigquery-rs:test".into()),
            missing_value: None,
        }
    }

    /// Batches `rows` as the writer does: a row that does not fit closes the open batch.
    fn batch_all(batcher: &mut Batcher, rows: &[usize]) -> Result<Vec<Batch>, CodecError> {
        let mut sealed = Vec::new();
        for &len in rows {
            let cost = batcher.cost(len)?;
            if !batcher.fits(cost) {
                sealed.extend(batcher.seal());
            }
            batcher.push(vec![7u8; len], cost, Instant::now());
            if batcher.is_full() {
                sealed.extend(batcher.seal());
            }
        }
        sealed.extend(batcher.seal());
        Ok(sealed)
    }

    proptest! {
        #[test]
        fn requests_never_exceed_the_budget(
            max_request_bytes in 2_000usize..40_000,
            columns in 1usize..40,
            rows in proptest::collection::vec(0usize..3_000, 1..200),
            max_rows in proptest::option::of(1usize..20),
        ) {
            let plan = plan(columns);
            let target = target();
            let mut batcher = Batcher::new(plan.clone(), &target, max_request_bytes, max_rows);
            let capacity = batcher.capacity();
            let fitting: Vec<usize> = rows.iter().copied().filter(|&len| len + 8 <= capacity).collect();
            prop_assume!(!fitting.is_empty());
            let batches = batch_all(&mut batcher, &fitting).expect("every row fits");
            let mut written = Vec::new();
            for (k, batch) in batches.iter().enumerate() {
                prop_assert_eq!(batch.index, k as u64);
                prop_assert_eq!(batch.first_row, written.len() as u64);
                if let Some(max_rows) = max_rows {
                    prop_assert!(batch.row_count() <= max_rows as u64);
                }
                let request = append_request(&target, Some(i64::MAX), &batch.rows, true);
                let size = request.encoded_len();
                prop_assert!(size <= max_request_bytes, "{} > {}", size, max_request_bytes);
                prop_assert_eq!(size - batch.bytes <= batcher.overhead(), true);
                written.extend(proto_rows(batch).iter().map(Vec::len));
            }
            prop_assert_eq!(written, fitting.clone());
            // No batch was closed early: the next batch's first row did not fit it.
            for pair in batches.windows(2) {
                let next = proto_rows(&pair[1])[0].len();
                let full_by_rows = max_rows.is_some_and(|m| pair[0].row_count() >= m as u64);
                let cost = 1 + encoded_len_varint(next as u64) + next;
                prop_assert!(full_by_rows || pair[0].bytes + cost > capacity);
            }
        }
    }

    #[test]
    fn a_row_larger_than_the_budget_is_row_too_large() {
        let target = target();
        let batcher = Batcher::new(plan(3), &target, 1_000, None);
        let capacity = batcher.capacity();
        let fits = capacity - 1 - encoded_len_varint(capacity as u64);
        assert!(batcher.cost(fits).is_ok());
        let err = batcher
            .cost(fits + 1)
            .expect_err("one byte over the budget");
        assert_eq!(err.kind(), BigQueryCodecErrorKind::RowTooLarge);
        let err = batcher.cost(10_000).expect_err("far over the budget");
        assert_eq!(err.kind(), BigQueryCodecErrorKind::RowTooLarge);
    }

    #[test]
    fn descriptor_size_is_reserved_in_every_request() {
        let target = target();
        let wide = plan(200);
        let budget = 30_000;
        let mut batcher = Batcher::new(wide.clone(), &target, budget, None);
        let descriptor = wide.descriptor().encoded_len();
        assert!(
            batcher.overhead() > descriptor,
            "{} <= {descriptor}",
            batcher.overhead()
        );
        let batches = batch_all(&mut batcher, &[500; 200]).expect("rows fit");
        assert!(batches.len() > 2);
        // Any batch may be the first one on a new connection, so each must fit with the
        // descriptor even when it was first sent without one.
        for batch in &batches {
            let with = append_request(&target, Some(1), &batch.rows, true);
            let without = append_request(&target, Some(1), &batch.rows, false);
            assert!(with.encoded_len() <= budget, "{}", with.encoded_len());
            assert!(without.encoded_len() + descriptor <= budget);
            assert!(writer_schema_is_absent(&without));
        }
    }

    fn proto_rows(batch: &Batch) -> &[Vec<u8>] {
        match &batch.rows {
            BatchRows::Proto { rows, .. } => rows,
            BatchRows::Arrow { .. } => panic!("the batcher seals protobuf rows"),
        }
    }

    /// Whether a request carries no writer schema, as every request after the first on a
    /// connection with an unchanged plan.
    fn writer_schema_is_absent(request: &AppendRowsRequest) -> bool {
        match &request.rows {
            Some(append_rows_request::Rows::ProtoRows(data)) => data.writer_schema.is_none(),
            _ => false,
        }
    }
}