use std::collections::HashMap;
use std::io::{BufRead, BufReader};
use tippecanoe::csv::{csv_dequote, csv_split, is_number};
use tippecanoe::errors::{EXIT_OPEN, EXIT_UTF8};
use tippecanoe::mvt::{MVT_DOUBLE, MVT_NULL, MVT_STRING, is_integer, is_unsigned_integer};
use tippecanoe::text::check_utf8_raw;
use super::tile::Val;
use crate::{Error, Result};
pub struct Field {
pub column: usize,
pub attr_type: i32,
pub value: Val<'static>,
pub sv_type: i32,
pub text: String,
pub number: f64,
warns: bool,
}
impl Field {
pub fn warn(&self) {
if self.warns {
eprintln!(
"Warning: numeric value {} could not be represented",
cstr(&self.text)
);
}
}
}
pub struct Row {
pub fields: Vec<Field>,
pub too_long: bool,
}
#[derive(Default)]
pub struct Csv {
pub header: Vec<String>,
pub mapping: HashMap<String, Row>,
}
fn cstr(s: &str) -> &str {
match s.find('\0') {
Some(n) => &s[..n],
None => s,
}
}
fn strtof(s: &str) -> (f32, usize) {
let c = std::ffi::CString::new(cstr(s)).unwrap_or_default();
let mut endptr: *mut libc::c_char = std::ptr::null_mut();
let f = unsafe { libc::strtof(c.as_ptr(), &mut endptr) };
(f, endptr as usize - c.as_ptr() as usize)
}
fn joined_value(type_: i32, s: &str) -> (Val<'static>, bool) {
let s = cstr(s);
if type_ != MVT_DOUBLE {
return (Val::Str(s.to_string().into()), false);
}
let mut v: i64 = 0;
let mut uv: u64 = 0;
if is_unsigned_integer(s, &mut uv) {
if uv <= i64::MAX as u64 {
return (Val::Int(uv as i64), false);
}
return (Val::Uint(uv), false);
}
if is_integer(s, &mut v) {
return (Val::Sint(v), false);
}
tippecanoe::cstd::set_errno(0);
let (f, endptr) = strtof(s);
if endptr == 0 || (f == f32::INFINITY && tippecanoe::cstd::errno() == libc::ERANGE) {
let (d, endptr) = tippecanoe::cstd::strtod(s);
let warns =
endptr == 0 || (d == f64::INFINITY && tippecanoe::cstd::errno() == libc::ERANGE);
return (Val::Double(d), warns);
}
let d = tippecanoe::cstd::atof(s);
if f as f64 == d {
(Val::Float(f), false)
} else {
(Val::Double(d), false)
}
}
#[derive(Clone, Debug, Default)]
pub struct RawCsv {
pub header: Vec<String>,
pub rows: HashMap<String, Vec<String>>,
}
impl RawCsv {
pub fn read(path: &str) -> Result<RawCsv> {
let f = std::fs::File::open(path).map_err(|e| {
Error::new(
EXIT_OPEN,
format!("{}: {}", path, crate::options::os_message(&e)),
)
})?;
let mut f = BufReader::new(f);
let mut csv = RawCsv::default();
let utf8_error = |err: &[u8]| {
let mut msg = format!("{}: ", path).into_bytes();
msg.extend_from_slice(err);
Error::new(EXIT_UTF8, String::from_utf8_lossy(&msg).into_owned())
};
let mut first = true;
loop {
let mut s = Vec::new();
let _ = f.read_until(b'\n', &mut s);
if s.is_empty() {
break;
}
let err = check_utf8_raw(&s);
if !err.is_empty() {
return Err(utf8_error(&err));
}
let s = String::from_utf8_lossy(&s);
if first {
first = false;
csv.header = csv_split(&s).into_iter().map(csv_dequote).collect();
continue;
}
let mut line = csv_split(&s);
if !line.is_empty() {
line[0] = csv_dequote(std::mem::take(&mut line[0]));
}
if !line.is_empty() && !csv.header.is_empty() && !csv.rows.contains_key(&line[0]) {
csv.rows.insert(line[0].clone(), line);
}
}
Ok(csv)
}
pub fn prepare(&self, empty_is_null: bool) -> Csv {
let header = &self.header;
let mut mapping = HashMap::with_capacity(self.rows.len());
for (key, line) in &self.rows {
let mut fields = Vec::new();
for (i, raw) in line
.iter()
.enumerate()
.skip(1)
.take(header.len().saturating_sub(1))
{
let mut joinval = raw.clone();
let mut attr_type = MVT_STRING;
if !joinval.is_empty() {
if joinval.as_bytes()[0] == b'"' {
joinval = csv_dequote(joinval);
} else if is_number(&joinval) {
attr_type = MVT_DOUBLE;
}
} else if empty_is_null {
attr_type = MVT_NULL;
}
let sv_type = if attr_type == MVT_STRING {
MVT_STRING
} else {
MVT_DOUBLE
};
let (value, warns) = joined_value(sv_type, &joinval);
fields.push(Field {
column: i,
attr_type,
value,
sv_type,
number: tippecanoe::cstd::atof(&joinval),
text: joinval,
warns,
});
}
let too_long = line.len() > header.len();
mapping.insert(key.clone(), Row { fields, too_long });
}
Csv {
header: header.clone(),
mapping,
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use tippecanoe::mvt::stringified_to_mvt_value;
#[test]
fn joined_values_as_tippecanoe_makes_them() {
for s in [
"0",
"1",
"-1",
"1.5",
"1.50",
"3.0000",
"-0",
"1e5",
"1e40",
"1e400",
"-1e400",
"18446744073709551615",
"9223372036854775807",
"9223372036854775808",
"-9223372036854775808",
"0.1",
"123456789.123",
" 7",
"abc",
"",
] {
for t in [MVT_DOUBLE, MVT_STRING] {
let ours = joined_value(t, s).0.to_mvt();
let theirs = stringified_to_mvt_value(t, s);
assert_eq!(ours.type_, theirs.type_, "{:?} as {}", s, t);
assert!(
ours == theirs || (ours.to_double().is_nan() && theirs.to_double().is_nan()),
"{:?} as {}",
s,
t
);
}
}
}
}