sqlload 0.70.2

SQL data loading tooling.
//! How reads of an ODBC source's tables would fetch their rows on this machine, from the driver's description of their columns.

use arrow_odbc::arrow_schema_from;
use arrow_odbc::odbc_api::Connection;

use crate::odbc_source::_helpers::batches::fetch_schema;
use crate::odbc_source::_helpers::columns::{fetch_shape, fetch_size};
use crate::odbc_source::_helpers::connect::connect;
use crate::odbc_source::_helpers::memory::{budget_note, fetch_budget, memory_line};
use crate::odbc_source::_helpers::query::select_query;
use crate::odbc_source::errors::OdbcSourceError;
use crate::odbc_source::models::{
    FetchShape, OdbcReadRequest, OdbcReader, PlannedFetch, ValueLimits,
};

/// For each of `requests`, in order, how its rows would be fetched: under the budget of `set` (the source's `max_batch_memory`), else 512 MiB lowered to a quarter of each table's `share` of the run's memory budget; in bulk or value by value, as the driver describes the prepared `SELECT` (without its filter). Nothing is run. All are described over one connection; when it can't be made, or the driver can't describe a read, its `fetch` line says why.
pub async fn plan_fetches(
    reader: &OdbcReader,
    requests: &[OdbcReadRequest<'_>],
    (set, share): (Option<usize>, Option<u64>),
) -> Vec<PlannedFetch> {
    let budget = fetch_budget(set, share);
    let note = budget_note(&budget);
    let queries: Vec<String> = requests
        .iter()
        .map(|request| {
            select_query(&OdbcReadRequest {
                filter: None,
                parameters: &[],
                ..*request
            })
        })
        .collect();
    let reader = reader.clone();
    let described = tokio::task::spawn_blocking(move || describe_all(&reader, &queries)).await;
    let described: Vec<Result<FetchShape, String>> = match described {
        Ok(Ok(shapes)) => shapes,
        Ok(Err(error)) => vec![Err(error.to_string()); requests.len()],
        Err(error) => vec![Err(format!("the ODBC description stopped: {error}")); requests.len()],
    };
    let mut planned: Vec<PlannedFetch> = Vec::with_capacity(requests.len());
    for (shape, request) in described.into_iter().zip(requests) {
        let (estimate, memory) = match shape {
            Ok(shape) => {
                let size = fetch_size(&shape, budget.bytes, request.batch_rows);
                (Some(size.bytes), memory_line(&shape, &size))
            }
            Err(error) => (
                None,
                format!("unknown: the driver doesn't describe the columns ({error})"),
            ),
        };
        planned.push(PlannedFetch {
            budget: budget.bytes,
            estimate,
            memory,
            budget_note: note.clone(),
        });
    }
    planned
}

/// The shape of each of `queries`' reads, over one connection.
fn describe_all(
    reader: &OdbcReader,
    queries: &[String],
) -> Result<Vec<Result<FetchShape, String>>, OdbcSourceError> {
    let connection = connect(reader)?;
    let dbms: String = connection.database_management_system_name()?;
    let mut shapes: Vec<Result<FetchShape, String>> = Vec::with_capacity(queries.len());
    for query in queries {
        shapes.push(
            described(&connection, (&dbms, query), &reader.limits)
                .map_err(|error| error.to_string()),
        );
    }
    Ok(shapes)
}

/// The shape of `query`'s read as the driver describes it prepared.
fn described(
    connection: &Connection<'_>,
    (dbms, query): (&str, &str),
    limits: &ValueLimits,
) -> Result<FetchShape, OdbcSourceError> {
    let mut prepared = connection.prepare(query)?;
    let schema = fetch_schema(&arrow_schema_from(&mut prepared, Some(dbms), false)?);
    fetch_shape(&mut prepared, &schema, limits)
}