use std::sync::Arc;
use datafusion::arrow::array::{Array, ArrayRef, AsArray, BooleanArray, Float64Array, NullArray, StringArray};
use datafusion::arrow::datatypes::{
DataType, Field, Float16Type, Float32Type, Float64Type, Int16Type, Int32Type, Int64Type, Int8Type, UInt16Type,
UInt32Type, UInt64Type, UInt8Type,
};
use datafusion::error::{DataFusionError, Result};
use fv_value::Value;
use geo_traits::{
CoordTrait, Dimensions, GeometryCollectionTrait, GeometryTrait, GeometryType, LineStringTrait,
MultiLineStringTrait, MultiPointTrait, MultiPolygonTrait, PointTrait, PolygonTrait,
};
pub fn has_value_form(t: &DataType) -> bool {
match t {
DataType::Null
| DataType::Boolean
| DataType::Int8
| DataType::Int16
| DataType::Int32
| DataType::Int64
| DataType::UInt8
| DataType::UInt16
| DataType::UInt32
| DataType::UInt64
| DataType::Float16
| DataType::Float32
| DataType::Float64
| DataType::Utf8
| DataType::LargeUtf8
| DataType::Utf8View => true,
DataType::Dictionary(_, values) => has_value_form(values),
DataType::List(f) | DataType::LargeList(f) | DataType::FixedSizeList(f, _) => has_value_form(f.data_type()),
DataType::Struct(fields) => fields.iter().all(|f| has_value_form(f.data_type())),
_ => false,
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum Geo {
Point,
Wkb,
}
fn geo_of(field: &Field) -> Option<Geo> {
let name = field.metadata().get("ARROW:extension:name")?;
match (name.as_str(), field.data_type()) {
("geoarrow.point", DataType::Struct(fs))
if fs.len() >= 2 && fs.iter().all(|f| f.data_type() == &DataType::Float64) =>
{
Some(Geo::Point)
}
("geoarrow.point", DataType::FixedSizeList(f, 2..=4)) if f.data_type() == &DataType::Float64 => {
Some(Geo::Point)
}
("geoarrow.wkb" | "ogc.wkb", DataType::Binary | DataType::LargeBinary | DataType::BinaryView) => Some(Geo::Wkb),
_ => None,
}
}
pub(crate) fn is_geoarrow(field: &Field) -> bool {
geo_of(field).is_some()
}
pub(crate) fn is_geoarrow_wkb(field: &Field) -> bool {
geo_of(field) == Some(Geo::Wkb)
}
pub fn field_has_value_form(field: &Field) -> bool {
geo_of(field).is_some() || has_value_form(field.data_type())
}
pub fn cell_to_value(array: &ArrayRef, i: usize) -> Result<Value> {
Ok(Cells::of(array)?.value(i))
}
#[derive(Clone, Copy, Debug)]
pub struct Cells<'a> {
array: &'a ArrayRef,
geo: Option<Geo>,
}
impl<'a> Cells<'a> {
pub fn of(array: &'a ArrayRef) -> Result<Self> {
if has_value_form(array.data_type()) {
Ok(Cells { array, geo: None })
} else {
Err(DataFusionError::Execution(format!(
"fv-value: a {} value has no form in the dialect",
array.data_type()
)))
}
}
pub fn of_field(field: &Field, array: &'a ArrayRef) -> Result<Self> {
match geo_of(field) {
Some(geo) => Ok(Cells { array, geo: Some(geo) }),
None => Cells::of(array),
}
}
#[inline]
pub fn value(&self, i: usize) -> Value {
match self.geo {
None => value_at(self.array, i),
Some(_) if self.array.is_null(i) => Value::Null,
Some(Geo::Point) => point_at(self.array, i),
Some(Geo::Wkb) => wkb_at(self.array, i),
}
}
}
fn point_at(array: &ArrayRef, i: usize) -> Value {
let coords: Vec<f64> = match array.data_type() {
DataType::Struct(fields) => {
let s = array.as_struct();
fields
.iter()
.zip(s.columns())
.filter(|(f, _)| f.name() != "m")
.map(|(_, c)| c.as_primitive::<Float64Type>().value(i))
.collect()
}
DataType::FixedSizeList(f, n) => {
let xs = array.as_fixed_size_list().value(i);
let xs = xs.as_primitive::<Float64Type>();
let take = if f.name().ends_with('m') {
*n as usize - 1
} else {
*n as usize
};
xs.values()[..take].to_vec()
}
_ => return Value::Null,
};
Value::List(coords.into_iter().map(Value::Num).collect())
}
fn wkb_at(array: &ArrayRef, i: usize) -> Value {
let bytes: &[u8] = match array.data_type() {
DataType::Binary => array.as_binary::<i32>().value(i),
DataType::LargeBinary => array.as_binary::<i64>().value(i),
DataType::BinaryView => array.as_binary_view().value(i),
_ => return Value::Null,
};
wkb::reader::read_wkb(bytes)
.ok()
.and_then(|g| geojson_of(&g))
.map_or(Value::Null, |j| fv_value::json_to_value(&j))
}
#[derive(Clone, Copy)]
struct Header {
le: bool,
ty: u32,
dim: usize,
at: usize,
}
#[inline]
fn header(b: &[u8]) -> Option<Header> {
let le = match *b.first()? {
1 => true,
0 => false,
_ => return None,
};
let raw = u32_at(b, 1, le)?;
let (ty, z) = match (raw & 0x0FFF_FFFF, raw & 0xC000_0000) {
(t @ 1..=7, 0) => (t, false),
(t @ 1001..=1007, 0) => (t - 1000, true),
(t @ 1..=7, 0x8000_0000) => (t, true),
_ => return None,
};
Some(Header {
le,
ty,
dim: if z { 3 } else { 2 },
at: if raw & 0x2000_0000 != 0 { 9 } else { 5 },
})
}
#[inline]
fn u32_at(b: &[u8], at: usize, le: bool) -> Option<u32> {
let w: [u8; 4] = b.get(at..at.checked_add(4)?)?.try_into().ok()?;
Some(if le {
u32::from_le_bytes(w)
} else {
u32::from_be_bytes(w)
})
}
#[inline]
fn f64_at(b: &[u8], at: usize, le: bool) -> f64 {
let w: [u8; 8] = b[at..at + 8].try_into().unwrap();
if le {
f64::from_le_bytes(w)
} else {
f64::from_be_bytes(w)
}
}
#[inline]
pub(crate) fn wkb_point(b: &[u8]) -> Option<(f64, f64, Option<f64>)> {
let h = header(b)?;
if h.ty != 1 || b.len() != h.at + 8 * h.dim {
return None;
}
let f = |k: usize| f64_at(b, h.at + 8 * k, h.le);
Some((f(0), f(1), (h.dim == 3).then(|| f(2))))
}
pub(crate) fn wkb_polygons(b: &[u8]) -> Option<WkbPolygons<'_>> {
let h = header(b)?;
let (count, rest) = match h.ty {
3 => (1, b),
6 => (u32_at(b, h.at, h.le)?, b.get(h.at + 4..)?),
_ => return None,
};
let mut at = 0;
for _ in 0..count {
let part = header(rest.get(at..)?)?;
if part.ty != 3 {
return None;
}
at += part.at + polygon_len(rest.get(at + part.at..)?, part)?;
}
(at == rest.len()).then_some(WkbPolygons { left: count, rest })
}
fn polygon_len(b: &[u8], h: Header) -> Option<usize> {
let rings = u32_at(b, 0, h.le)?;
let mut at = 4usize;
for _ in 0..rings {
let n = u32_at(b, at, h.le)? as usize;
at = at.checked_add(4)?.checked_add(n.checked_mul(8 * h.dim)?)?;
if at > b.len() {
return None;
}
}
Some(at)
}
pub(crate) struct WkbPolygons<'a> {
left: u32,
rest: &'a [u8],
}
impl<'a> Iterator for WkbPolygons<'a> {
type Item = WkbRings<'a>;
#[inline]
fn next(&mut self) -> Option<WkbRings<'a>> {
self.left = self.left.checked_sub(1)?;
let h = header(self.rest)?;
let body = &self.rest[h.at..];
let len = polygon_len(body, h)?;
self.rest = &body[len..];
Some(WkbRings {
le: h.le,
dim: h.dim,
left: u32_at(body, 0, h.le)?,
rest: &body[4..len],
})
}
}
pub(crate) struct WkbRings<'a> {
le: bool,
dim: usize,
left: u32,
rest: &'a [u8],
}
impl<'a> Iterator for WkbRings<'a> {
type Item = WkbPositions<'a>;
#[inline]
fn next(&mut self) -> Option<WkbPositions<'a>> {
self.left = self.left.checked_sub(1)?;
let n = u32_at(self.rest, 0, self.le)? as usize;
let (ring, rest) = self.rest[4..].split_at(n * 8 * self.dim);
self.rest = rest;
Some(WkbPositions {
le: self.le,
step: 8 * self.dim,
bytes: ring,
})
}
}
pub(crate) struct WkbPositions<'a> {
le: bool,
step: usize,
bytes: &'a [u8],
}
impl Iterator for WkbPositions<'_> {
type Item = geo3d::Geodetic;
#[inline]
fn next(&mut self) -> Option<geo3d::Geodetic> {
if self.bytes.len() < self.step {
return None;
}
let p = geo3d::Geodetic::from_lon_lat(f64_at(self.bytes, 0, self.le), f64_at(self.bytes, 8, self.le), 0.0);
self.bytes = &self.bytes[self.step..];
Some(p)
}
}
fn geojson_of(g: &impl GeometryTrait<T = f64>) -> Option<serde_json::Value> {
use serde_json::{json, Value as J};
fn position(c: &impl CoordTrait<T = f64>) -> Option<J> {
let n = match c.dim() {
Dimensions::Xy | Dimensions::Xym => 2,
Dimensions::Xyz | Dimensions::Xyzm => 3,
Dimensions::Unknown(_) => return None,
};
Some((0..n).map(|k| c.nth_or_panic(k)).collect())
}
fn line(l: &impl LineStringTrait<T = f64>) -> Option<J> {
l.coords().map(|c| position(&c)).collect()
}
fn rings(p: &impl PolygonTrait<T = f64>) -> Option<J> {
p.exterior()
.into_iter()
.chain(p.interiors())
.map(|r| line(&r))
.collect()
}
let (kind, coordinates) = match g.as_type() {
GeometryType::Point(p) => ("Point", position(&p.coord()?)?),
GeometryType::LineString(l) => ("LineString", line(l)?),
GeometryType::Polygon(p) => ("Polygon", rings(p)?),
GeometryType::MultiPoint(m) => (
"MultiPoint",
m.points().map(|p| position(&p.coord()?)).collect::<Option<J>>()?,
),
GeometryType::MultiLineString(m) => (
"MultiLineString",
m.line_strings().map(|l| line(&l)).collect::<Option<J>>()?,
),
GeometryType::MultiPolygon(m) => ("MultiPolygon", m.polygons().map(|p| rings(&p)).collect::<Option<J>>()?),
GeometryType::GeometryCollection(c) => {
let geometries: Option<J> = c.geometries().map(|g| geojson_of(&g)).collect();
return Some(json!({"type": "GeometryCollection", "geometries": geometries?}));
}
_ => return None,
};
Some(json!({"type": kind, "coordinates": coordinates}))
}
pub(crate) fn value_at(array: &ArrayRef, i: usize) -> Value {
if array.is_null(i) {
return Value::Null;
}
match array.data_type() {
DataType::Null => Value::Null,
DataType::Boolean => Value::Bool(array.as_boolean().value(i)),
DataType::Float64 => Value::Num(array.as_primitive::<Float64Type>().value(i)),
DataType::Float32 => Value::Num(f64::from(array.as_primitive::<Float32Type>().value(i))),
DataType::Float16 => Value::Num(f64::from(array.as_primitive::<Float16Type>().value(i))),
DataType::Int64 => Value::Num(array.as_primitive::<Int64Type>().value(i) as f64),
DataType::Int32 => Value::Num(f64::from(array.as_primitive::<Int32Type>().value(i))),
DataType::Int16 => Value::Num(f64::from(array.as_primitive::<Int16Type>().value(i))),
DataType::Int8 => Value::Num(f64::from(array.as_primitive::<Int8Type>().value(i))),
DataType::UInt64 => Value::Num(array.as_primitive::<UInt64Type>().value(i) as f64),
DataType::UInt32 => Value::Num(f64::from(array.as_primitive::<UInt32Type>().value(i))),
DataType::UInt16 => Value::Num(f64::from(array.as_primitive::<UInt16Type>().value(i))),
DataType::UInt8 => Value::Num(f64::from(array.as_primitive::<UInt8Type>().value(i))),
DataType::Utf8 => Value::Str(array.as_string::<i32>().value(i).to_string()),
DataType::LargeUtf8 => Value::Str(array.as_string::<i64>().value(i).to_string()),
DataType::Utf8View => Value::Str(array.as_string_view().value(i).to_string()),
DataType::Dictionary(_, _) => {
let dict = array.as_any_dictionary();
value_at(dict.values(), dict.normalized_keys()[i])
}
DataType::List(_) => list(&array.as_list::<i32>().value(i)),
DataType::LargeList(_) => list(&array.as_list::<i64>().value(i)),
DataType::FixedSizeList(_, _) => list(&array.as_fixed_size_list().value(i)),
DataType::Struct(fields) => {
let s = array.as_struct();
Value::Obj(
fields
.iter()
.zip(s.columns())
.map(|(f, c)| (f.name().clone(), value_at(c, i)))
.collect(),
)
}
_ => Value::Null,
}
}
fn list(items: &ArrayRef) -> Value {
Value::List((0..items.len()).map(|j| value_at(items, j)).collect())
}
pub fn data_type_for(kind: Kind) -> DataType {
match kind {
Kind::Num => DataType::Float64,
Kind::Str => DataType::Utf8,
Kind::Bool => DataType::Boolean,
Kind::Null => DataType::Null,
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum Kind {
Num,
Str,
Bool,
Null,
}
impl Kind {
pub fn of_data_type(t: &DataType) -> Option<Kind> {
match t {
DataType::Null => Some(Kind::Null),
DataType::Boolean => Some(Kind::Bool),
DataType::Int8
| DataType::Int16
| DataType::Int32
| DataType::Int64
| DataType::UInt8
| DataType::UInt16
| DataType::UInt32
| DataType::UInt64
| DataType::Float32
| DataType::Float64 => Some(Kind::Num),
DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View => Some(Kind::Str),
DataType::Dictionary(_, values) => Kind::of_data_type(values),
_ => None,
}
}
}
pub fn values_to_array(values: &[Value], data_type: &DataType) -> Result<ArrayRef> {
let wrong = |v: &Value| {
DataFusionError::Execution(format!(
"fv-value: expression produced a {} where the plan declared {data_type}",
v.type_name()
))
};
Ok(match data_type {
DataType::Float64 => Arc::new(
values
.iter()
.map(|v| match v {
Value::Null => Ok(None),
Value::Num(n) => Ok(Some(*n)),
other => Err(wrong(other)),
})
.collect::<Result<Float64Array>>()?,
),
DataType::Utf8 => Arc::new(
values
.iter()
.map(|v| match v {
Value::Null => Ok(None),
Value::Str(s) => Ok(Some(s.clone())),
other => Err(wrong(other)),
})
.collect::<Result<StringArray>>()?,
),
DataType::Boolean => Arc::new(
values
.iter()
.map(|v| match v {
Value::Null => Ok(None),
Value::Bool(b) => Ok(Some(*b)),
other => Err(wrong(other)),
})
.collect::<Result<BooleanArray>>()?,
),
DataType::Null => Arc::new(NullArray::new(values.len())),
other => {
return Err(DataFusionError::NotImplemented(format!(
"fv-value: cannot materialise dialect values as {other}"
)))
}
})
}
#[cfg(test)]
mod tests {
use super::*;
use datafusion::arrow::array::{Int32Array, ListBuilder, StringViewArray};
#[test]
fn cells_convert_by_type() {
let ints: ArrayRef = Arc::new(Int32Array::from(vec![Some(7), None]));
assert_eq!(cell_to_value(&ints, 0).unwrap(), Value::Num(7.0));
assert_eq!(cell_to_value(&ints, 1).unwrap(), Value::Null);
let views: ArrayRef = Arc::new(StringViewArray::from(vec!["x"]));
assert_eq!(cell_to_value(&views, 0).unwrap(), Value::from("x"));
let mut lb = ListBuilder::new(Float64Array::builder(2));
lb.values().append_value(1.0);
lb.values().append_value(2.0);
lb.append(true);
let list: ArrayRef = Arc::new(lb.finish());
assert_eq!(cell_to_value(&list, 0).unwrap(), Value::from(vec![1.0, 2.0]));
}
#[test]
fn cells_check_the_type_once_and_refuse_one_with_no_dialect_value() {
use datafusion::arrow::array::BinaryArray;
let words: ArrayRef = Arc::new(StringViewArray::from(vec!["a", "b"]));
let cells = Cells::of(&words).unwrap();
assert_eq!((cells.value(0), cells.value(1)), (Value::from("a"), Value::from("b")));
let bytes: ArrayRef = Arc::new(BinaryArray::from(vec![b"x".as_ref()]));
let err = Cells::of(&bytes).unwrap_err().to_string();
assert!(
err.contains("Binary") && err.contains("no form in the dialect"),
"{err}"
);
assert!(cell_to_value(&bytes, 0).is_err());
assert!(!has_value_form(&DataType::Date32) && has_value_form(&DataType::Float16));
}
#[test]
fn kinds_and_materialisation() {
assert_eq!(Kind::of_data_type(&DataType::Int64), Some(Kind::Num));
assert_eq!(Kind::of_data_type(&DataType::Utf8View), Some(Kind::Str));
assert_eq!(Kind::of_data_type(&DataType::Date32), None);
let a = values_to_array(&[Value::Num(1.5), Value::Null], &DataType::Float64).unwrap();
assert_eq!(a.len(), 2);
assert!(a.is_null(1));
assert!(values_to_array(&[Value::from("s")], &DataType::Float64).is_err());
assert_eq!(data_type_for(Kind::Bool), DataType::Boolean);
}
}