use arrow_schema::{DataType, SchemaRef, TimeUnit};
use parquet::arrow::arrow_reader::ArrowReaderMetadata;
use parquet::basic::{
ConvertedType, LogicalType, TimeUnit as ParquetTimeUnit, Type as PhysicalType,
};
use parquet::schema::types::Type as ParquetType;
use crate::column_types::main::portable_type::portable_type;
use crate::column_types::models::ColumnType;
use crate::files_source::constants::{LOADER_COLUMNS, MAX_DIRECT_DECIMAL_PRECISION};
use crate::files_source::models::{FileColumn, StoredRead};
/// A column seen in sampled files: its type, or `None` while every sampled value was NULL.
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct SampledColumn {
pub(crate) name: String,
pub(crate) column_type: Option<Result<ColumnType, String>>,
/// How the column is held when a direct `COPY` does not load it exactly.
pub(crate) direct_refusal: Option<String>,
pub(crate) stored_read: StoredRead,
/// Whether the sampled values were lists (`unnest = "all"` unnests the column): `Some(true)` when some held items and every other was NULL or empty, `Some(false)` once any was not a list, `None` while none decided.
pub(crate) lists: Option<bool>,
/// The header this column repeats (see [`FileColumn::repeat_of`](crate::files_source::models::FileColumn::repeat_of)).
pub(crate) repeat_of: Option<String>,
}
/// What two sightings of a column's values say about it being a list field: any value that is not a list says no, any list with items yes.
pub(crate) fn joined_lists(seen: Option<bool>, found: Option<bool>) -> Option<bool> {
match (seen, found) {
(Some(false), _) | (_, Some(false)) => Some(false),
(Some(true), _) | (_, Some(true)) => Some(true),
(None, None) => None,
}
}
/// What one value of JSON text `text` says about its column being a list field: `None` for NULL or an empty list.
pub(crate) fn text_lists(text: &str) -> Option<bool> {
let text = text.trim_start();
match text.as_bytes().first() {
None | Some(b'n') => None,
Some(b'[') => (!text[1..].trim_start().starts_with(']')).then_some(true),
Some(_) => Some(false),
}
}
/// What an Arrow column of `data_type` says about being a list field.
fn arrow_lists(data_type: &DataType) -> Option<bool> {
Some(matches!(
data_type,
DataType::List(_) | DataType::LargeList(_) | DataType::FixedSizeList(..)
))
}
/// Whether `name` is one of the loader's own columns, ignoring case.
pub(crate) fn loader_column(name: &str) -> bool {
LOADER_COLUMNS
.iter()
.any(|reserved| reserved.eq_ignore_ascii_case(name))
}
/// `found` without the loader's own columns, and the names of those dropped (as the file spells them).
pub(crate) fn drop_loader_columns(found: Vec<SampledColumn>) -> (Vec<SampledColumn>, Vec<String>) {
let (dropped, kept): (Vec<SampledColumn>, Vec<SampledColumn>) = found
.into_iter()
.partition(|column| loader_column(&column.name));
(
kept,
dropped.into_iter().map(|column| column.name).collect(),
)
}
/// The columns of a Parquet schema.
pub(crate) fn schema_columns(schema: &SchemaRef) -> Vec<SampledColumn> {
schema
.fields()
.iter()
.map(|field| SampledColumn {
name: field.name().clone(),
column_type: Some(portable_type(field)),
direct_refusal: direct_refusal(field.data_type()),
stored_read: arrow_read(field.data_type()),
lists: arrow_lists(field.data_type()),
repeat_of: None,
})
.collect()
}
/// The top-level field `name` of a Parquet schema.
fn stored_field<'a>(
fields: &'a [std::sync::Arc<ParquetType>],
name: &str,
) -> Option<&'a ParquetType> {
fields
.iter()
.find(|field| field.name() == name)
.map(AsRef::as_ref)
}
/// The columns of a Parquet footer, each checked for the direct route against what the file itself stores as well as its Arrow type: plan samples these, and each batch's check reads the same.
pub(crate) fn parquet_columns(footer: &ArrowReaderMetadata) -> Vec<SampledColumn> {
let stored = footer
.metadata()
.file_metadata()
.schema_descr()
.root_schema()
.get_fields();
footer
.schema()
.fields()
.iter()
.map(|field| {
let (direct_refusal, stored_read) =
match stored_read(field.data_type(), stored_field(stored, field.name())) {
Ok(read) => (direct_refusal(field.data_type()), read),
Err(refusal) => (Some(refusal), StoredRead::Plain),
};
SampledColumn {
name: field.name().clone(),
column_type: Some(portable_type(field)),
direct_refusal,
stored_read,
lists: arrow_lists(field.data_type()),
repeat_of: None,
}
})
.collect()
}
/// How a stage read returns a column of Arrow type `data_type`, from the type alone (nanoseconds or not).
pub(crate) fn arrow_read(data_type: &DataType) -> StoredRead {
match data_type {
DataType::Timestamp(TimeUnit::Nanosecond, _) | DataType::Time64(TimeUnit::Nanosecond) => {
StoredRead::Nanos
}
DataType::Dictionary(_, value) => arrow_read(value),
_ => StoredRead::Plain,
}
}
/// How a stage read returns column `stored`, which an embedded Arrow schema reads as `data_type`, or why a direct `COPY` cannot load it exactly (decision 96; live, 2026-11-05). With `USE_LOGICAL_TYPE`, every timestamp, time and date the file annotates reads exactly, nanoseconds and INT96 included (INT96 only where SQLLoad reads it at nanoseconds too: no embedded type, or one of nanoseconds); a bare INT64 under an Arrow timestamp or `Date64` is converted from its Arrow unit; any other column without the logical type its Arrow type needs is refused.
pub(crate) fn stored_read(
data_type: &DataType,
stored: Option<&ParquetType>,
) -> Result<StoredRead, String> {
let logical = stored.and_then(|column| column.get_basic_info().logical_type_ref());
let physical = stored
.filter(|column| {
column.is_primitive() && column.get_basic_info().converted_type() == ConvertedType::NONE
})
.map(ParquetType::get_physical_type);
let read = match (data_type, logical, physical) {
(DataType::Dictionary(_, value), _, _) => return stored_read(value, stored),
(DataType::Timestamp(..), Some(LogicalType::Timestamp(timestamp)), _)
| (DataType::Time32(_) | DataType::Time64(_), Some(LogicalType::Time(timestamp)), _) => {
match timestamp.unit {
ParquetTimeUnit::NANOS => Some(StoredRead::Nanos),
_ => Some(StoredRead::Plain),
}
}
(DataType::Timestamp(TimeUnit::Nanosecond, _), None, Some(PhysicalType::INT96)) => {
Some(StoredRead::Nanos)
}
(DataType::Timestamp(unit, _), None, Some(PhysicalType::INT64)) => {
Some(StoredRead::Epoch(match unit {
TimeUnit::Second => 0,
TimeUnit::Millisecond => 3,
TimeUnit::Microsecond => 6,
TimeUnit::Nanosecond => 9,
}))
}
(DataType::Date64, None, Some(PhysicalType::INT64)) => Some(StoredRead::Epoch(3)),
(DataType::Date32 | DataType::Date64, Some(LogicalType::Date), _) => {
Some(StoredRead::Plain)
}
(
DataType::Decimal32(..)
| DataType::Decimal64(..)
| DataType::Decimal128(..)
| DataType::Decimal256(..),
Some(LogicalType::Decimal { .. }),
_,
) => Some(StoredRead::Plain),
(
DataType::Timestamp(..)
| DataType::Time32(_)
| DataType::Time64(_)
| DataType::Date32
| DataType::Date64
| DataType::Decimal32(..)
| DataType::Decimal64(..)
| DataType::Decimal128(..)
| DataType::Decimal256(..),
_,
_,
) => None,
_ => Some(StoredRead::Plain),
};
read.ok_or_else(|| {
let held = match (logical, physical) {
(Some(logical), _) => format!("{logical:?}"),
(None, Some(physical)) => physical.to_string(),
(None, None) => match stored {
Some(column) if column.is_primitive() => {
column.get_basic_info().converted_type().to_string()
}
_ => "nothing".to_owned(),
},
};
format!("{data_type} stored as {held}")
})
}
/// `data_type` as text unless a direct `COPY` is known to load it exactly (decision 96; live, 2026-11-05): `Float16` makes Snowflake reject the whole file, a decimal over 38 digits does not fit, nested types load as JSON text only through SQLLoad's reader, and other types not listed have no portable type yet.
pub(crate) fn direct_refusal(data_type: &DataType) -> Option<String> {
let exact = match data_type {
DataType::Boolean
| DataType::Int8
| DataType::Int16
| DataType::Int32
| DataType::Int64
| DataType::UInt8
| DataType::UInt16
| DataType::UInt32
| DataType::UInt64
| DataType::Float32
| DataType::Float64
| DataType::Utf8
| DataType::LargeUtf8
| DataType::Utf8View
| DataType::Binary
| DataType::LargeBinary
| DataType::BinaryView
| DataType::FixedSizeBinary(_)
| DataType::Date32
| DataType::Date64
| DataType::Time32(_)
| DataType::Time64(_)
| DataType::Timestamp(..) => true,
DataType::Decimal32(precision, _)
| DataType::Decimal64(precision, _)
| DataType::Decimal128(precision, _)
| DataType::Decimal256(precision, _) => *precision <= MAX_DIRECT_DECIMAL_PRECISION,
DataType::Dictionary(_, value) => return direct_refusal(value),
_ => false,
};
let nested = matches!(
data_type,
DataType::List(_)
| DataType::LargeList(_)
| DataType::FixedSizeList(..)
| DataType::Struct(_)
| DataType::Map(..)
);
match (exact, nested) {
(true, _) => None,
(false, true) => Some(format!(
"{data_type} (a nested type, which loads as JSON only through SQLLoad's reader; leave the column out with exclude_columns)"
)),
(false, false) => Some(data_type.to_string()),
}
}
/// How one file's columns join those of newer files.
#[derive(Debug, Clone, Copy)]
pub(crate) struct ColumnMerge<'a> {
/// The file, for problems.
pub(crate) key: &'a str,
/// Columns whose values come from the path, so the file's own are ignored.
pub(crate) replaced: &'a [&'a str],
/// Whether an int column in one file and a float column in another make a float column (JSON and CSV, whose types are inferred) rather than a problem.
pub(crate) widen_numbers: bool,
/// `infer_types = false`: scalar columns load as text, so files may disagree on their types.
pub(crate) as_text: bool,
}
/// `columns` plus `found`'s new columns, and a problem for each column whose type differs from an earlier file's.
pub(crate) fn merge_columns(
mut columns: Vec<SampledColumn>,
found: Vec<SampledColumn>,
merge: ColumnMerge<'_>,
) -> (Vec<SampledColumn>, Vec<String>) {
let mut problems: Vec<String> = Vec::new();
for column in found {
if merge.replaced.contains(&column.name.as_str()) {
continue;
}
let Some(existing) = columns.iter_mut().find(|seen| seen.name == column.name) else {
columns.push(column);
continue;
};
if existing.direct_refusal.is_none() {
existing.direct_refusal = match joined_read(existing.stored_read, column.stored_read) {
Some(_) => column.direct_refusal,
None => Some(format!(
"{} stored as a bare integer in some files only (in {} or newer ones)",
column.name, merge.key
)),
};
}
existing.stored_read =
joined_read(existing.stored_read, column.stored_read).unwrap_or(existing.stored_read);
existing.lists = joined_lists(existing.lists, column.lists);
existing.repeat_of = existing.repeat_of.take().or(column.repeat_of);
match (&existing.column_type, column.column_type) {
(_, None) => {}
(None, found_type) => existing.column_type = found_type,
(Some(seen), Some(found_type)) if *seen == found_type => {}
(Some(Ok(seen)), Some(Ok(found_type)))
if merge.widen_numbers && is_number(seen) && is_number(&found_type) =>
{
existing.column_type = Some(Ok(ColumnType::Float));
}
(Some(Ok(seen)), Some(Ok(found_type)))
if merge.as_text && ![seen, &found_type].contains(&&ColumnType::Json) =>
{
existing.column_type = Some(Ok(ColumnType::Text));
}
(Some(seen), Some(found_type)) => problems.push(format!(
"column {} is {} in newer files but {} in {}; pin a type that holds both",
column.name,
describe(seen),
describe(&found_type),
merge.key
)),
}
}
(columns, problems)
}
/// How a column read two ways in different files is read in all of them: nanoseconds where any file has them (a cast keeps shorter values whole), and `None` when only some files hold bare integers, which take their own conversion.
pub(crate) fn joined_read(seen: StoredRead, found: StoredRead) -> Option<StoredRead> {
match (seen, found) {
(seen, found) if seen == found => Some(seen),
(StoredRead::Epoch(_), _) | (_, StoredRead::Epoch(_)) => None,
_ => Some(StoredRead::Nanos),
}
}
/// A problem for each pair of columns whose names differ only in case, which most warehouses cannot hold apart.
pub(crate) fn case_problems(columns: &[SampledColumn]) -> Vec<String> {
let mut problems: Vec<String> = Vec::new();
for (position, column) in columns.iter().enumerate() {
let clash = columns[..position]
.iter()
.find(|earlier| earlier.name.eq_ignore_ascii_case(&column.name));
if let Some(earlier) = clash {
problems.push(format!(
"columns {} and {} differ only in case, which most warehouses cannot hold apart; rename one in the files",
earlier.name, column.name
));
}
}
problems
}
fn is_number(column_type: &ColumnType) -> bool {
matches!(column_type, ColumnType::Int | ColumnType::Float)
}
/// `columns` plus `schema`'s new columns (except `replaced` ones), and a problem for each column whose type differs from an earlier file's.
pub(crate) fn merge_schema(
columns: Vec<FileColumn>,
schema: &SchemaRef,
key: &str,
replaced: &[&str],
) -> (Vec<FileColumn>, Vec<String>) {
let seen: Vec<SampledColumn> = columns
.into_iter()
.map(|column| SampledColumn {
name: column.name,
column_type: Some(column.column_type),
direct_refusal: column.direct_refusal,
stored_read: column.stored_read,
lists: column.lists,
repeat_of: column.repeat_of,
})
.collect();
let merge = ColumnMerge {
key,
replaced,
widen_numbers: false,
as_text: false,
};
let (merged, problems) = merge_columns(seen, schema_columns(schema), merge);
(file_columns(merged), problems)
}
/// The sampled columns with their final types: a column that was NULL in every sampled value is text.
pub(crate) fn file_columns(columns: Vec<SampledColumn>) -> Vec<FileColumn> {
columns
.into_iter()
.map(|column| FileColumn {
name: column.name,
column_type: column.column_type.unwrap_or(Ok(ColumnType::Text)),
direct_refusal: column.direct_refusal,
stored_read: column.stored_read,
lists: column.lists,
repeat_of: column.repeat_of,
})
.collect()
}
fn describe(column_type: &Result<ColumnType, String>) -> String {
match column_type {
Ok(column_type) => column_type.to_string(),
Err(_) => "unsupported".to_owned(),
}
}