datafusion-server 0.21.0

Web server library for session-based queries using Arrow and other large datasets as data sources.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28

// infer_schema.rs - Infer schema from data sources
// Sasaki, Naoki <nsasaki@sal.co.jp> March 28, 2023
//

use datafusion::arrow;
use serde_json::Value;

use crate::request::body::DataSourceOption;
use crate::response::http_error::ResponseError;

pub fn from_json_value(
    json_rows: &[Value],
    options: &DataSourceOption,
) -> anyhow::Result<arrow::datatypes::Schema> {
    let infer_schema_rows =
        std::cmp::min(options.infer_schema_rows.unwrap_or(100), json_rows.len());
    infer_schema(json_rows, infer_schema_rows)
}

fn infer_schema(
    json_rows: &[Value],
    max_infer_rows: usize,
) -> anyhow::Result<arrow::datatypes::Schema> {
    Ok(arrow::json::reader::infer_json_schema_from_iterator(
        json_rows[0..max_infer_rows].iter().map(|v| Ok(v.clone())),
    )
    .map_err(ResponseError::record_batch_creation)?)
}