use arrow_array::RecordBatch;
use arrow_ipc::writer::{IpcWriteOptions, StreamWriter};
use arrow_ipc::CompressionType;
use arrow_schema::{ArrowError, Schema};
use gcloud_sdk::google::cloud::bigquery::storage::v1 as storage;
use gcloud_sdk::google::cloud::bigquery::storage::v1::storage_error::StorageErrorCode;
use gcloud_sdk::google::cloud::bigquery::storage::v1::StorageError;
use gcloud_sdk::google::cloud::bigquery::v2;
use gcloud_sdk::google::rpc::Status;
use gcloud_sdk::prost::Message;
use gcloud_sdk::tonic::Code;
#[derive(Clone, Copy, Debug, Default)]
pub(crate) struct IpcCompression(Option<CompressionType>);
impl From<storage::arrow_serialization_options::CompressionCodec> for IpcCompression {
fn from(codec: storage::arrow_serialization_options::CompressionCodec) -> Self {
use storage::arrow_serialization_options::CompressionCodec;
Self(match codec {
CompressionCodec::Lz4Frame => Some(CompressionType::LZ4_FRAME),
CompressionCodec::Zstd => Some(CompressionType::ZSTD),
CompressionCodec::CompressionUnspecified => None,
})
}
}
impl From<v2::arrow_serialization_options::CompressionCodec> for IpcCompression {
fn from(codec: v2::arrow_serialization_options::CompressionCodec) -> Self {
use v2::arrow_serialization_options::CompressionCodec;
Self(match codec {
CompressionCodec::Lz4Frame => Some(CompressionType::LZ4_FRAME),
CompressionCodec::Zstd => Some(CompressionType::ZSTD),
CompressionCodec::CompressionUnspecified => None,
})
}
}
pub(crate) struct IpcMessages {
pub schema: Vec<u8>,
pub batches: Vec<Vec<u8>>,
}
impl IpcMessages {
pub(crate) fn encode(
schema: &Schema,
batches: &[RecordBatch],
compression: IpcCompression,
) -> Result<Self, ArrowError> {
let options = IpcWriteOptions::default().try_with_compression(compression.0)?;
let mut writer = StreamWriter::try_new_with_options(Vec::new(), schema, options)?;
let schema = writer.get_ref().clone();
let batches = batches
.iter()
.map(|batch| {
let start = writer.get_ref().len();
writer.write(batch)?;
Ok(writer.get_ref()[start..].to_vec())
})
.collect::<Result<_, ArrowError>>()?;
Ok(Self { schema, batches })
}
}
pub(crate) fn storage_error_status(
code: Code,
storage: StorageErrorCode,
entity: &str,
message: &str,
) -> Status {
Status {
code: code as i32,
message: message.into(),
details: vec![gcloud_sdk::prost_types::Any {
type_url: "type.googleapis.com/google.cloud.bigquery.storage.v1.StorageError".into(),
value: StorageError {
code: storage.into(),
entity: entity.into(),
error_message: message.into(),
}
.encode_to_vec(),
}],
}
}