use std::collections::HashMap;
use crate::row_batches::types::ListChoices;
use arrow_array::RecordBatch;
use crate::column_parse::models::{BadValue, ColumnParse};
use crate::load_naming::main::loaded_name::loaded_name;
use crate::load_naming::models::{Compat, Naming};
use crate::row_batches::constants::FLATTEN_SEPARATOR;
#[derive(Debug, Clone, PartialEq)]
pub enum Cell {
Null,
Bool(bool),
Int(i64),
Float(f64),
Number(String),
Text(String),
Json(String),
Date(i32),
TimestampNtz(i64),
Decimal(String),
}
#[derive(Debug, Default)]
pub struct RowColumns {
names: Vec<String>,
positions: HashMap<String, usize>,
cells: Vec<Vec<Cell>>,
rows: usize,
guess: usize,
flattened: Vec<String>,
}
impl RowColumns {
pub fn new() -> Self {
Self::default()
}
pub fn push(&mut self, name: &str, cell: Cell) {
let position = match self.names.get(self.guess) {
Some(guessed) if guessed == name => self.guess,
_ => self.position(name),
};
self.guess = position + 1;
let column = &mut self.cells[position];
column.resize(self.rows, Cell::Null);
column.push(cell);
}
pub fn end_row(&mut self) {
self.rows += 1;
self.guess = 0;
}
pub fn approximate_bytes(&self) -> usize {
self.cells
.iter()
.flatten()
.map(|cell| {
std::mem::size_of::<Cell>()
+ match cell {
Cell::Number(text)
| Cell::Text(text)
| Cell::Json(text)
| Cell::Decimal(text) => text.len(),
Cell::Null
| Cell::Bool(_)
| Cell::Int(_)
| Cell::Float(_)
| Cell::Date(_)
| Cell::TimestampNtz(_) => 0,
}
})
.sum()
}
pub fn set_flattened(&mut self, names: Vec<String>) {
self.flattened = names;
}
pub fn flattened(&self) -> &[String] {
&self.flattened
}
pub fn rows(&self) -> usize {
self.rows
}
pub fn into_columns(self) -> (usize, Vec<(String, Vec<Cell>)>) {
let rows = self.rows;
let columns: Vec<(String, Vec<Cell>)> = self
.names
.into_iter()
.zip(self.cells)
.map(|(name, mut cells)| {
cells.resize(rows, Cell::Null);
(name, cells)
})
.collect();
(rows, columns)
}
fn position(&mut self, name: &str) -> usize {
match self.positions.get(name) {
Some(position) => *position,
None => {
self.names.push(name.to_owned());
self.positions.insert(name.to_owned(), self.names.len() - 1);
self.cells.push(Vec::new());
self.names.len() - 1
}
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum JsonLayout {
Document,
Lines,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct JsonRead {
pub layout: JsonLayout,
pub path: Option<String>,
pub batch_rows: usize,
pub projection: Option<Projection>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Projection {
pub fields: Vec<ProjectedField>,
pub zone: Option<chrono_tz::Tz>,
pub keep_record: Option<String>,
pub tree: PathNode,
}
#[derive(Debug, Clone, PartialEq, Eq, Default)]
pub struct PathNode {
pub children: HashMap<String, PathNode>,
pub ends: Vec<usize>,
pub under: Vec<usize>,
}
#[derive(Debug, Clone, Copy, Default)]
pub struct ProjectionRules<'a> {
pub cursor_column: Option<&'a str>,
pub flatten: Option<&'a Flatten>,
pub masked: &'a [String],
pub naming: (Naming, Option<usize>),
}
#[derive(Debug, Clone, PartialEq, Eq, serde::Deserialize)]
pub struct FieldSpec {
pub name: String,
pub path: String,
#[serde(default)]
pub required: bool,
#[serde(default)]
pub parse: Option<serde_json::Value>,
#[serde(default)]
pub on_error: BadValue,
#[serde(default)]
pub from: FieldFrom,
}
#[derive(Debug, Clone, PartialEq, Eq, Default, serde::Deserialize)]
#[serde(tag = "kind", rename_all = "snake_case")]
pub enum FieldFrom {
#[default]
Row,
Parent {
levels: usize,
},
Key {
name: String,
},
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ProjectedField {
pub name: String,
pub path: Vec<String>,
pub text: String,
pub required: bool,
pub parse: Option<ColumnParse>,
}
#[derive(Debug, Default)]
pub struct JsonWriter {
text: String,
}
impl JsonWriter {
pub fn new() -> Self {
Self::default()
}
pub fn raw(&mut self, text: &str) {
self.text.push_str(text);
}
pub fn string(&mut self, text: &str) {
self.text.push('"');
let mut start = 0;
for (position, byte) in text.bytes().enumerate() {
let escape = match byte {
b'"' => "\\\"",
b'\\' => "\\\\",
b'\n' => "\\n",
b'\r' => "\\r",
b'\t' => "\\t",
0x08 => "\\b",
0x0c => "\\f",
0x00..=0x1f => "",
_ => continue,
};
self.text.push_str(&text[start..position]);
match escape.is_empty() {
true => self.text.push_str(&format!("\\u{byte:04x}")),
false => self.text.push_str(escape),
}
start = position + 1;
}
self.text.push_str(&text[start..]);
self.text.push('"');
}
pub fn float(&mut self, value: f64) {
match serde_json::Number::from_f64(value) {
Some(number) => self.text.push_str(&number.to_string()),
None => self.text.push_str("null"),
}
}
pub fn take(&mut self) -> String {
std::mem::take(&mut self.text)
}
}
#[derive(Debug, Clone, PartialEq, Eq, Default)]
pub enum Unnest {
#[default]
None,
All,
Paths(Vec<String>),
}
impl Unnest {
pub fn takes(&self, name: &str, choices: &ListChoices) -> bool {
match self {
Self::Paths(paths) => paths.iter().any(|path| path == name),
Self::All => choices.get(name) == Some(&true),
Self::None => false,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Default)]
pub enum FlattenObjects {
#[default]
None,
All,
Paths(Vec<String>),
}
#[derive(Debug, Clone, PartialEq, Eq, Default)]
pub struct Flatten {
pub objects: FlattenObjects,
pub keep: Vec<String>,
pub pinned: Vec<(String, String)>,
pub max_depth: u32,
pub naming: Naming,
pub excluded: ExcludePatterns,
pub as_text: bool,
pub nonfinite_null: bool,
}
#[derive(Debug, Clone, Default)]
pub struct ExcludePatterns {
patterns: Vec<String>,
set: globset::GlobSet,
}
impl PartialEq for ExcludePatterns {
fn eq(&self, other: &Self) -> bool {
self.patterns == other.patterns
}
}
impl Eq for ExcludePatterns {}
impl ExcludePatterns {
pub fn new(patterns: Vec<String>) -> Result<Self, globset::Error> {
let mut builder = globset::GlobSetBuilder::new();
for pattern in &patterns {
builder.add(globset::Glob::new(pattern)?);
}
let set = builder.build()?;
Ok(Self { patterns, set })
}
pub fn bypassed(&self, name: &str) -> Option<&str> {
let below = self.set.matches(format!("{name}{FLATTEN_SEPARATOR}"));
self.set
.matches(name)
.into_iter()
.find(|index| !below.contains(index))
.map(|index| self.patterns[index].as_str())
}
}
impl Flatten {
pub fn is_none(&self) -> bool {
self.objects == FlattenObjects::None
}
pub fn flattens(&self, path: &[&str]) -> Result<bool, String> {
let chosen = match &self.objects {
FlattenObjects::None => false,
FlattenObjects::All => true,
FlattenObjects::Paths(_) => self.lists(path),
};
if !chosen || path.len() > self.max_depth as usize || self.keeps(path) {
return Ok(false);
}
let name = path.join(FLATTEN_SEPARATOR);
if let Some(pattern) = self.excluded.bypassed(&name) {
return Err(format!(
"exclude_columns {pattern} names {name}, which flatten replaces with {name}__<key> columns the pattern doesn't match; name those columns instead, or keep {name} whole with columns = {{ {name} = \"json\" }}"
));
}
match self
.pinned
.iter()
.find(|(pinned, _)| self.same_parts(pinned.split(FLATTEN_SEPARATOR), path))
{
Some((pinned, column_type)) => Err(format!(
"columns {pinned} = {column_type:?}: flatten replaces {name} with its keys' columns; pin it \"json\" to keep it whole"
)),
None => Ok(true),
}
}
pub fn lists(&self, path: &[&str]) -> bool {
match &self.objects {
FlattenObjects::Paths(paths) => paths
.iter()
.any(|listed| self.same_parts(listed.split('.'), path)),
FlattenObjects::None | FlattenObjects::All => false,
}
}
pub fn may_make(&self, name: &str) -> bool {
match &self.objects {
FlattenObjects::None => false,
FlattenObjects::All => name.contains(FLATTEN_SEPARATOR),
FlattenObjects::Paths(paths) => paths.iter().any(|listed| {
let prefix = format!(
"{}{FLATTEN_SEPARATOR}",
listed.replace('.', FLATTEN_SEPARATOR)
);
name.get(..prefix.len())
.is_some_and(|start| start.eq_ignore_ascii_case(&prefix))
}),
}
}
pub fn pins_array(&self, path: &[&str]) -> bool {
self.pinned.iter().any(|(pinned, column_type)| {
column_type
.trim()
.to_ascii_lowercase()
.starts_with("array(")
&& self.same_parts(pinned.split(FLATTEN_SEPARATOR), path)
})
}
pub fn keeps(&self, path: &[&str]) -> bool {
self.keep
.iter()
.any(|name| self.same_parts(name.split(FLATTEN_SEPARATOR), path))
}
fn same_parts<'a>(&self, parts: impl Iterator<Item = &'a str>, path: &[&str]) -> bool {
let parts: Vec<&str> = parts.collect();
parts.len() == path.len()
&& parts.iter().zip(path).all(|(part, key)| {
part == key
|| (self.naming == Naming::SnakeCase
&& loaded_name(self.naming, part, None)
== loaded_name(self.naming, key, None))
})
}
}
#[derive(Debug, Clone, Copy)]
pub struct UnnestRequest<'a> {
pub table: &'a str,
pub unnest: &'a Unnest,
pub max_depth: u32,
pub naming: Naming,
pub compat: Compat,
pub name_limit: Option<usize>,
pub flatten: &'a Flatten,
pub live: &'a [String],
pub ids: &'a [String],
pub roots: Option<&'a [String]>,
}
#[derive(Debug)]
pub struct Unnested {
pub parent: RecordBatch,
pub children: Vec<ChildBatch>,
}
#[derive(Debug)]
pub struct ChildBatch {
pub path: String,
pub table: String,
pub batch: RecordBatch,
pub root_rows: Vec<usize>,
}