sqlload 0.4.0

SQL data loading tooling. Pre-release, under development.
//! Stream files into Arrow batches shaped for the destination.

use arrow_array::RecordBatch;
use futures_util::TryStreamExt;
use parquet::arrow::ParquetRecordBatchStreamBuilder;

use crate::file_store::models::OpenFolder;
use crate::files_source::_helpers::parquet::footers::object_reader;
use crate::files_source::_helpers::plan::encoding::file_codec;
use crate::files_source::_helpers::plan::shape::{output_schema, shape_batch};
use crate::files_source::_helpers::text::text_reader::TextReader;
use crate::files_source::errors::FilesError;
use crate::files_source::models::{Codec, FileRead, FilesRead, SourceFormat};

/// Read every file of `request` in order, passing each shaped batch to `on_batch`; returns the row count.
pub async fn read_files(
    folder: &OpenFolder,
    request: &FilesRead,
    mut on_batch: impl AsyncFnMut(RecordBatch) -> Result<(), String>,
) -> Result<u64, FilesError> {
    let schema = output_schema(&request.columns).map_err(|reason| FilesError::File {
        key: folder.location.to_string(),
        reason,
    })?;
    let mut rows: u64 = 0;
    for data in &request.files {
        let failed = |reason: String| FilesError::File {
            key: data.file.relative_key.clone(),
            reason,
        };
        let file = FileRead {
            folder,
            request,
            data,
            schema: &schema,
            file_name: format!("{}{}", folder.location, data.file.relative_key),
        };
        rows += match request.encoding.format {
            SourceFormat::Parquet => read_parquet(&file, &mut on_batch).await,
            _ => read_text(&file, &mut on_batch).await,
        }
        .map_err(failed)?;
    }
    Ok(rows)
}

async fn read_parquet(
    file: &FileRead<'_>,
    on_batch: &mut impl AsyncFnMut(RecordBatch) -> Result<(), String>,
) -> Result<u64, String> {
    let mut stream =
        ParquetRecordBatchStreamBuilder::new(object_reader(file.folder, &file.data.file))
            .await
            .and_then(|builder| builder.with_batch_size(file.request.batch_rows).build())
            .map_err(|error| error.to_string())?;
    let mut rows: u64 = 0;
    while let Some(batch) = stream.try_next().await.map_err(|error| error.to_string())? {
        let shaped = shape_batch(&batch, file.data, &file.file_name, file.schema)
            .map_err(|error| error.reason)?;
        rows += shaped.num_rows() as u64;
        on_batch(shaped).await?;
    }
    Ok(rows)
}

/// Read a JSON or CSV file; a value that does not convert names its line.
async fn read_text(
    file: &FileRead<'_>,
    on_batch: &mut impl AsyncFnMut(RecordBatch) -> Result<(), String>,
) -> Result<u64, String> {
    let codec =
        file_codec(&file.data.file.relative_key, &file.request.encoding).unwrap_or(Codec::Plain);
    let mut reader = TextReader::open(file.folder, &file.data.file, codec, file.request).await?;
    let mut rows: u64 = 0;
    while let Some((batch, lines)) = reader.next_batch().await? {
        let shaped =
            shape_batch(&batch, file.data, &file.file_name, file.schema).map_err(|error| {
                match error.row.and_then(|row| lines.get(row)) {
                    Some(line) => format!(
                        "line {line}: {}; fix the file, or pin a type that holds the value",
                        error.reason
                    ),
                    None => error.reason,
                }
            })?;
        rows += shaped.num_rows() as u64;
        on_batch(shaped).await?;
    }
    Ok(rows)
}