datui_lib/
column_types.rs

1//! A column's declared type: one vocabulary for the type row, a delimited spec's
2//! `[columns]`, the typing of text columns, and a column retyped in the table.
3//!
4//! [`ColumnType`] is a type name and, for the temporal ones, a strftime `format`, read
5//! from and written as the inline table a spec entry uses: `{ type = "date", format =
6//! "%d/%m/%Y" }`. [`ColumnType::expr`] turns a column into it, lazily and never failing:
7//! a value that does not fit is null. [`ColumnType::unfit`] counts those values, for
8//! the note that says so ([`unfit_notes`]).
9
10use polars::prelude::*;
11
12/// The type names, as the type row shows them and a spec or a picker takes them.
13pub const TYPE_NAMES: [&str; 16] = [
14    "str", "bool", "i8", "i16", "i32", "i64", "u8", "u16", "u32", "u64", "f32", "f64", "date",
15    "time", "datetime", "duration",
16];
17
18/// The type a name in [`TYPE_NAMES`] stands for.
19pub fn dtype_named(name: &str) -> Option<DataType> {
20    Some(match name {
21        "str" => DataType::String,
22        "bool" => DataType::Boolean,
23        "i8" => DataType::Int8,
24        "i16" => DataType::Int16,
25        "i32" => DataType::Int32,
26        "i64" => DataType::Int64,
27        "u8" => DataType::UInt8,
28        "u16" => DataType::UInt16,
29        "u32" => DataType::UInt32,
30        "u64" => DataType::UInt64,
31        "f32" => DataType::Float32,
32        "f64" => DataType::Float64,
33        "date" => DataType::Date,
34        "time" => DataType::Time,
35        "datetime" => DataType::Datetime(TimeUnit::Microseconds, None),
36        "duration" => DataType::Duration(TimeUnit::Nanoseconds),
37        _ => return None,
38    })
39}
40
41/// The short name of a column's type, as the type row and the schema pane spell it.
42///
43/// Polars' own `Display` says `Datetime(Microseconds, None)`; the row under the header
44/// has room for one word.
45pub fn dtype_label(dtype: &DataType) -> String {
46    match dtype {
47        DataType::String => "str".to_string(),
48        DataType::Boolean => "bool".to_string(),
49        DataType::Int8 => "i8".to_string(),
50        DataType::Int16 => "i16".to_string(),
51        DataType::Int32 => "i32".to_string(),
52        DataType::Int64 => "i64".to_string(),
53        DataType::UInt8 => "u8".to_string(),
54        DataType::UInt16 => "u16".to_string(),
55        DataType::UInt32 => "u32".to_string(),
56        DataType::UInt64 => "u64".to_string(),
57        DataType::Float32 => "f32".to_string(),
58        DataType::Float64 => "f64".to_string(),
59        DataType::Date => "date".to_string(),
60        DataType::Datetime(_, _) => "datetime".to_string(),
61        DataType::Time => "time".to_string(),
62        DataType::Duration(_) => "duration".to_string(),
63        DataType::Binary => "binary".to_string(),
64        DataType::Null => "null".to_string(),
65        DataType::List(inner) => format!("list[{}]", dtype_label(inner)),
66        DataType::Struct(_) => "struct".to_string(),
67        other if other.is_categorical() => "cat".to_string(),
68        other if other.is_enum() => "enum".to_string(),
69        other if other.is_decimal() => "decimal".to_string(),
70        other => other.to_string().to_ascii_lowercase(),
71    }
72}
73
74/// A column's declared type.
75#[derive(Debug, Clone, PartialEq, Eq)]
76pub struct ColumnType {
77    pub dtype: DataType,
78    /// The strftime format of the text, for `date`, `time` and `datetime`; inferred
79    /// from the values when not given.
80    pub format: Option<String>,
81}
82
83/// Written as the inline table a spec entry is, in a saved view's JSON too: `{ "type":
84/// "date", "format": "%d/%m/%Y" }`.
85impl serde::Serialize for ColumnType {
86    fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
87        use serde::ser::SerializeMap;
88        let mut map = serializer.serialize_map(None)?;
89        map.serialize_entry("type", &self.name())?;
90        if let Some(format) = &self.format {
91            map.serialize_entry("format", format)?;
92        }
93        map.end()
94    }
95}
96
97impl<'de> serde::Deserialize<'de> for ColumnType {
98    fn deserialize<D: serde::Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
99        #[derive(serde::Deserialize)]
100        #[serde(deny_unknown_fields)]
101        struct Entry {
102            #[serde(rename = "type")]
103            name: String,
104            #[serde(default)]
105            format: Option<String>,
106        }
107        let entry = Entry::deserialize(deserializer)?;
108        Self::named(&entry.name, entry.format).map_err(serde::de::Error::custom)
109    }
110}
111
112/// A change to a view's columns, by name: a type, or a column made from others, as a
113/// spec's `[columns]` entry says it.
114#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
115pub struct ColumnChange {
116    pub name: String,
117    #[serde(flatten)]
118    pub change: Change,
119}
120
121#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
122#[serde(untagged)]
123pub enum Change {
124    /// `{ from = ["Lcl Date", "Lcl Time", "UTCOfst"], as = "datetime" }`.
125    Made {
126        from: Vec<String>,
127        #[serde(rename = "as")]
128        kind: String,
129        #[serde(default, skip_serializing_if = "Option::is_none")]
130        format: Option<String>,
131    },
132    /// `{ type = "i64" }`.
133    Typed(ColumnType),
134}
135
136impl ColumnChange {
137    /// The spec entry it is: `zip = { type = "str" }`.
138    pub fn to_toml(&self) -> String {
139        let value = match &self.change {
140            Change::Typed(ty) => ty.to_toml(),
141            Change::Made { from, kind, format } => {
142                let mut table = toml_edit::InlineTable::new();
143                let mut list = toml_edit::Array::new();
144                for f in from {
145                    list.push(f.as_str());
146                }
147                table.insert("from", toml_edit::Value::Array(list));
148                table.insert("as", kind.as_str().into());
149                if let Some(format) = format {
150                    table.insert("format", format.as_str().into());
151                }
152                table.fmt();
153                table.to_string()
154            }
155        };
156        let mut key = toml_edit::Key::new(self.name.as_str());
157        key.fmt();
158        format!("{} = {value}", key.display_repr())
159    }
160
161    /// The derived column it makes, for a change that makes one.
162    pub fn derived(&self) -> Option<Derived> {
163        let Change::Made { from, kind, format } = &self.change else {
164            return None;
165        };
166        Some(Derived {
167            name: self.name.clone(),
168            from: from.clone(),
169            kind: DerivedKind::named(kind)?,
170            format: format.clone(),
171        })
172    }
173}
174
175/// The characters trimmed from a value before it is read as a type.
176const PADDING: &str = " \t\r\n";
177
178impl ColumnType {
179    /// The type `name` names, with `format` for a temporal one. The error says what
180    /// was wrong in words a spec's error or a prompt can carry.
181    pub fn named(name: &str, format: Option<String>) -> Result<Self, String> {
182        let dtype = dtype_named(name).ok_or_else(|| {
183            format!(
184                "unknown type \"{name}\"; expected one of {}",
185                TYPE_NAMES.join(", ")
186            )
187        })?;
188        let ty = Self { dtype, format };
189        if ty.format.is_some() && !ty.is_temporal() {
190            return Err(format!(
191                "format is for date, time and datetime, not {}",
192                ty.name()
193            ));
194        }
195        Ok(ty)
196    }
197
198    pub fn name(&self) -> String {
199        dtype_label(&self.dtype)
200    }
201
202    pub fn is_temporal(&self) -> bool {
203        matches!(
204            self.dtype,
205            DataType::Date | DataType::Time | DataType::Datetime(_, _)
206        )
207    }
208
209    fn is_integer(&self) -> bool {
210        self.dtype.is_integer()
211    }
212
213    /// The inline table a spec entry and a saved view write it as.
214    pub fn to_toml(&self) -> String {
215        let mut table = toml_edit::InlineTable::new();
216        table.insert("type", self.name().into());
217        if let Some(format) = &self.format {
218            table.insert("format", format.as_str().into());
219        }
220        table.fmt();
221        table.to_string()
222    }
223
224    /// The type an inline table such as `{ type = "date", format = "%d/%m/%Y" }` says.
225    pub fn from_toml(text: &str) -> Result<Self, String> {
226        let value: toml_edit::Value = text.parse().map_err(|e| format!("{e}"))?;
227        let table = value
228            .as_inline_table()
229            .ok_or_else(|| "expected an inline table such as { type = \"i64\" }".to_string())?;
230        for (key, _) in table.iter() {
231            if key != "type" && key != "format" {
232                return Err(format!("unknown key `{key}`; expected type or format"));
233            }
234        }
235        let name = table
236            .get("type")
237            .and_then(|v| v.as_str())
238            .ok_or_else(|| "missing `type`".to_string())?;
239        let format = match table.get("format") {
240            Some(v) => Some(
241                v.as_str()
242                    .ok_or_else(|| "format: expected a string".to_string())?
243                    .to_string(),
244            ),
245            None => None,
246        };
247        Self::named(name, format)
248    }
249
250    fn strptime(&self) -> StrptimeOptions {
251        StrptimeOptions {
252            format: self.format.as_deref().map(PlSmallStr::from),
253            strict: false,
254            exact: true,
255            cache: true,
256        }
257    }
258
259    /// `text`, a text expression, trimmed and with a blank value null.
260    fn trimmed(text: Expr) -> Expr {
261        let trimmed = text
262            .str()
263            .strip_chars(lit(PlSmallStr::from_static(PADDING)));
264        when(trimmed.clone().eq(lit(PlSmallStr::from_static(""))))
265            .then(Null {}.lit().cast(DataType::String))
266            .otherwise(trimmed)
267    }
268
269    /// The column `name`, now of type `from`, as this type. Text is trimmed first, and a
270    /// blank value is null. `bool` takes `true`/`false` and `1`/`0`, in any case. A value
271    /// that does not fit, or is out of range, is null: the read never fails on one.
272    pub fn expr(&self, name: &str, from: &DataType) -> Expr {
273        self.typed(name, from).alias(PlSmallStr::from(name))
274    }
275
276    fn typed(&self, name: &str, from: &DataType) -> Expr {
277        let column = col(PlSmallStr::from(name));
278        if *from != DataType::String {
279            return column.cast(self.dtype.clone());
280        }
281        if self.dtype == DataType::String {
282            return column;
283        }
284        let text = Self::trimmed(column);
285        match &self.dtype {
286            DataType::Boolean => {
287                let lower = text.str().to_lowercase();
288                when(lower.clone().eq(lit("true")).or(lower.clone().eq(lit("1"))))
289                    .then(lit(true))
290                    .when(lower.clone().eq(lit("false")).or(lower.eq(lit("0"))))
291                    .then(lit(false))
292                    .otherwise(Null {}.lit().cast(DataType::Boolean))
293            }
294            DataType::Date => text.str().to_date(self.strptime()),
295            DataType::Time => text.str().to_time(self.strptime()),
296            DataType::Datetime(unit, zone) => text.str().to_datetime(
297                Some(*unit),
298                zone.clone(),
299                self.strptime(),
300                lit(PlSmallStr::from_static("raise")),
301            ),
302            DataType::Duration(_) => text.map(
303                |c: Column| Ok(durations(c.str()?).into_column()),
304                |_: &Schema, field: &Field| {
305                    Ok(Field::new(
306                        field.name().clone(),
307                        DataType::Duration(TimeUnit::Nanoseconds),
308                    ))
309                },
310            ),
311            dtype => text.cast(dtype.clone()),
312        }
313    }
314
315    /// The counts of the values of the column `name`, of type `from`, that [`Self::expr`]
316    /// makes null: those that do not parse, and those out of range for an integer type.
317    /// Two expressions summed over the column, named `{name}\0not` and `{name}\0range`.
318    pub fn unfit(&self, name: &str, from: &DataType) -> [Expr; 2] {
319        let column = col(PlSmallStr::from(name));
320        let typed = self.expr(name, from);
321        let (present, integral) = if *from == DataType::String {
322            let text = Self::trimmed(column);
323            let integral = text
324                .clone()
325                .str()
326                .contains(lit(r"^[+-]?[0-9]+$"), false)
327                .fill_null(lit(false));
328            (text.is_not_null(), integral)
329        } else {
330            (column.is_not_null(), lit(from.is_primitive_numeric()))
331        };
332        let unfit = present.and(typed.is_null());
333        let range = self.is_integer();
334        let not = if range {
335            unfit.clone().and(integral.clone().not())
336        } else {
337            unfit.clone()
338        };
339        let out = if range {
340            unfit.and(integral)
341        } else {
342            lit(false)
343        };
344        [
345            not.cast(DataType::UInt64)
346                .sum()
347                .alias(format!("{name}\u{0}not")),
348            out.cast(DataType::UInt64)
349                .sum()
350                .alias(format!("{name}\u{0}range")),
351        ]
352    }
353}
354
355/// The date formats `read.infer_types` tries and a retype offers, most common first.
356pub const DATE_FORMATS: &[&str] = &[
357    "%Y-%m-%d", "%Y/%m/%d", "%Y.%m.%d", "%Y%m%d", "%d-%m-%Y", "%d/%m/%Y", "%d.%m.%Y", "%m-%d-%Y",
358    "%m/%d/%Y",
359];
360
361/// The datetime formats, as [`DATE_FORMATS`].
362pub const DATETIME_FORMATS: &[&str] = &[
363    // ISO 8601 with an offset. `%#z` takes `Z`, `+05:00`, `-0500` and `+05`; a format
364    // with an offset makes Polars read the column into UTC.
365    "%Y-%m-%dT%H:%M:%S%.f%#z",
366    "%Y-%m-%d %H:%M:%S%.f%#z",
367    "%Y-%m-%dT%H:%M%#z",
368    "%Y-%m-%dT%H:%M:%S%.f",
369    "%Y-%m-%dT%H:%M:%S",
370    "%Y-%m-%dT%H:%M",
371    "%Y-%m-%d %H:%M:%S%.f",
372    "%Y-%m-%d %H:%M:%S",
373    "%Y-%m-%d %H:%M",
374    "%Y-%m-%d",
375    "%d-%m-%YT%H:%M:%S%.f",
376    "%d-%m-%YT%H:%M:%S",
377    "%d-%m-%Y %H:%M:%S%.f",
378    "%d-%m-%Y %H:%M:%S",
379    "%d/%m/%YT%H:%M:%S%.f",
380    "%d/%m/%YT%H:%M:%S",
381    "%d/%m/%Y %H:%M:%S",
382    "%Y%m%dT%H%M%S%.f",
383    "%Y%m%d %H%M%S",
384];
385
386/// The time formats, as [`DATE_FORMATS`].
387pub const TIME_FORMATS: &[&str] = &[
388    "%H:%M:%S%.9f",
389    "%H:%M:%S%.6f",
390    "%H:%M:%S%.3f",
391    "%H:%M:%S",
392    "%H:%M",
393];
394
395/// The formats of a temporal `dtype` that read `sample`, in the order they are tried.
396pub fn formats_reading(dtype: &DataType, sample: &str) -> Vec<&'static str> {
397    use chrono::{NaiveDate, NaiveDateTime, NaiveTime};
398    let sample = sample.trim();
399    match dtype {
400        DataType::Date => DATE_FORMATS
401            .iter()
402            .filter(|f| NaiveDate::parse_from_str(sample, f).is_ok())
403            .copied()
404            .collect(),
405        DataType::Datetime(_, _) => DATETIME_FORMATS
406            .iter()
407            .filter(|f| NaiveDateTime::parse_from_str(sample, f).is_ok())
408            .copied()
409            .collect(),
410        DataType::Time => TIME_FORMATS
411            .iter()
412            .filter(|f| NaiveTime::parse_from_str(sample, f).is_ok())
413            .copied()
414            .collect(),
415        _ => Vec::new(),
416    }
417}
418
419/// `value` read as `ty`, as the table would show it: `2024-03-04` for `03/04/2024` as a
420/// date with `%d/%m/%Y`. `None` when it does not read.
421pub fn preview(ty: &ColumnType, value: &str) -> Option<String> {
422    let df = df!("v" => [value]).ok()?;
423    let out = df
424        .lazy()
425        .select([ty.expr("v", &DataType::String)])
426        .collect()
427        .ok()?;
428    let v = out.column("v").ok()?.get(0).ok()?;
429    (!v.is_null()).then(|| match v {
430        AnyValue::String(s) => s.to_string(),
431        v => v.to_string(),
432    })
433}
434
435/// Text in Polars' duration format (`1d`, `2h30m`, `-1w2d`) as nanoseconds. A value that
436/// does not parse is null.
437pub fn durations(text: &StringChunked) -> DurationChunked {
438    let values: Vec<Option<i64>> = text
439        .iter()
440        .map(|v| {
441            v.and_then(|s| {
442                polars::time::Duration::try_parse(s)
443                    .ok()
444                    .map(|d| d.duration_ns())
445            })
446        })
447        .collect();
448    Int64Chunked::from_iter_options(text.name().clone(), values.into_iter())
449        .into_duration(TimeUnit::Nanoseconds)
450}
451
452/// Whether `value` starts with a zero another digit follows, as `02134`, `007` and
453/// `-01` do and `0`, `0.5`, `-0.5` and `0e3` do not: a code or an ID, kept as text
454/// rather than read as a number.
455pub fn has_leading_zero(value: &str) -> bool {
456    let digits = value.trim().trim_start_matches(['+', '-']).as_bytes();
457    digits.len() > 1 && digits[0] == b'0' && digits[1].is_ascii_digit()
458}
459
460/// A column given a type, and the type it had before: what [`unfit_frame`] counts.
461#[derive(Debug, Clone, PartialEq, Eq)]
462pub struct Typed {
463    pub column: String,
464    pub ty: ColumnType,
465    pub from: DataType,
466}
467
468/// One pass over `source`, the frame before the columns of `typed` were typed, that
469/// counts the values each one made null.
470pub fn unfit_frame(source: LazyFrame, typed: &[Typed]) -> LazyFrame {
471    let exprs: Vec<Expr> = typed
472        .iter()
473        .flat_map(|t| t.ty.unfit(&t.column, &t.from))
474        .collect();
475    source.select(exprs)
476}
477
478/// The values a column's type made null: those that are not of it, and those out of
479/// its range.
480#[derive(Debug, Clone, PartialEq, Eq)]
481pub struct Unfit {
482    pub column: String,
483    pub ty: String,
484    pub not_parsed: u64,
485    pub out_of_range: u64,
486}
487
488/// The counts in `counted`, the one row [`unfit_frame`] collects, for `typed`.
489pub fn unfit_counts(counted: &DataFrame, typed: &[Typed]) -> Vec<Unfit> {
490    let count = |name: String| -> u64 {
491        counted
492            .column(&name)
493            .ok()
494            .and_then(|c| c.get(0).ok())
495            .and_then(|v| v.extract::<u64>())
496            .unwrap_or(0)
497    };
498    typed
499        .iter()
500        .map(|t| Unfit {
501            column: t.column.clone(),
502            ty: t.ty.name(),
503            not_parsed: count(format!("{}\u{0}not", t.column)),
504            out_of_range: count(format!("{}\u{0}range", t.column)),
505        })
506        .filter(|u| u.not_parsed > 0 || u.out_of_range > 0)
507        .collect()
508}
509
510/// A note a column, for each column whose type made values null:
511/// `LogIdx: 3 values not i64, read as null`, `RPM: 2 values out of range for u8, read
512/// as null`.
513pub fn unfit_notes(unfit: &[Unfit], scope: &str) -> Vec<crate::notes::Note> {
514    let values = |n: u64| if n == 1 { "value" } else { "values" };
515    unfit
516        .iter()
517        .map(|u| {
518            let mut said = Vec::new();
519            if u.not_parsed > 0 {
520                said.push(format!(
521                    "{} {} not {}",
522                    u.not_parsed,
523                    values(u.not_parsed),
524                    u.ty
525                ));
526            }
527            if u.out_of_range > 0 {
528                said.push(format!(
529                    "{} {} out of range for {}",
530                    u.out_of_range,
531                    values(u.out_of_range),
532                    u.ty
533                ));
534            }
535            crate::notes::Note {
536                summary: format!("{}: {}, read as null", u.column, said.join(", ")),
537                scope: scope.to_string(),
538                read_as_text: None,
539                passed_over: None,
540            }
541        })
542        .collect()
543}
544
545/// What a derived column is.
546#[derive(Debug, Clone, Copy, PartialEq, Eq)]
547pub enum DerivedKind {
548    /// From a date and a time, or one text column, with an optional UTC offset.
549    Datetime,
550    Date,
551    Time,
552}
553
554impl DerivedKind {
555    pub const ALL: [Self; 3] = [Self::Datetime, Self::Date, Self::Time];
556
557    /// The kind `as` names.
558    pub fn named(name: &str) -> Option<Self> {
559        Self::ALL.into_iter().find(|k| k.name() == name)
560    }
561
562    pub fn name(self) -> &'static str {
563        match self {
564            Self::Datetime => "datetime",
565            Self::Date => "date",
566            Self::Time => "time",
567        }
568    }
569}
570
571/// A column built from others: `time = { from = ["Date", "Time", "Offset"], as = "datetime" }`.
572#[derive(Debug, Clone, PartialEq, Eq)]
573pub struct Derived {
574    pub name: String,
575    pub from: Vec<String>,
576    pub kind: DerivedKind,
577    /// A strftime format for the text the `from` columns make, joined with a space.
578    /// Inferred from the values when not given.
579    pub format: Option<String>,
580}
581
582/// An offset such as `-05:00`, `+0530`, `-5` or `05:00:00`: its sign, hours and minutes.
583const OFFSET: &str = r"^\s*([+-])?(\d{1,2})(?::?(\d{2}))?(?::\d{2})?\s*$";
584
585impl Derived {
586    /// The column, lazily, from its sources' text.
587    pub fn expr(&self) -> Expr {
588        let text = |name: &str| col(name).cast(DataType::String);
589        let options = StrptimeOptions {
590            format: self.format.as_deref().map(PlSmallStr::from),
591            strict: false,
592            exact: true,
593            cache: true,
594        };
595        match self.kind {
596            DerivedKind::Date => text(&self.from[0]).str().to_date(options),
597            DerivedKind::Time => text(&self.from[0]).str().to_time(options),
598            DerivedKind::Datetime => {
599                let stamp = match self.from.as_slice() {
600                    [one] => text(one),
601                    // A null in either is a null stamp.
602                    [date, time, ..] => text(date) + lit(" ") + text(time),
603                    [] => unreachable!("a derived column has a source"),
604                };
605                let local = stamp.str().to_datetime(
606                    Some(TimeUnit::Microseconds),
607                    None,
608                    options,
609                    lit("raise"),
610                );
611                let Some(offset) = self.from.get(2) else {
612                    return local;
613                };
614                // Local time less its offset from UTC is UTC.
615                let part = |group| text(offset).str().extract(lit(OFFSET), group);
616                let sign = when(part(1).eq(lit("-")))
617                    .then(lit(-1i64))
618                    .otherwise(lit(1i64));
619                let minutes = sign
620                    * (part(2).cast(DataType::Int64) * lit(60i64)
621                        + part(3).cast(DataType::Int64).fill_null(lit(0i64)));
622                let shift =
623                    (minutes * lit(60_000_000i64)).cast(DataType::Duration(TimeUnit::Microseconds));
624                (local - shift).dt().replace_time_zone(
625                    Some(polars::prelude::TimeZone::UTC),
626                    lit("raise"),
627                    NonExistent::Raise,
628                )
629            }
630        }
631    }
632}
633
634#[cfg(test)]
635mod tests {
636    use super::*;
637
638    fn typed(name: &str, format: Option<&str>, values: &[Option<&str>]) -> (Series, Vec<Unfit>) {
639        let ty = ColumnType::named(name, format.map(String::from)).unwrap();
640        let df = df!("v" => values).unwrap();
641        let out = df
642            .clone()
643            .lazy()
644            .select([ty.expr("v", &DataType::String).alias("v")])
645            .collect()
646            .unwrap();
647        let typed = [Typed {
648            column: "v".into(),
649            ty,
650            from: DataType::String,
651        }];
652        let counted = unfit_frame(df.lazy(), &typed).collect().unwrap();
653        (
654            out.column("v").unwrap().as_materialized_series().clone(),
655            unfit_counts(&counted, &typed),
656        )
657    }
658
659    #[test]
660    fn a_preview_shows_what_a_type_makes_of_a_value() {
661        let date = ColumnType::named("date", Some("%d/%m/%Y".into())).unwrap();
662        assert_eq!(preview(&date, "03/04/2024").as_deref(), Some("2024-04-03"));
663        assert_eq!(preview(&date, "2024-04-03"), None);
664    }
665
666    #[test]
667    fn a_leading_zero_is_a_code() {
668        for code in ["02134", "007", "-01", " 0012 "] {
669            assert!(has_leading_zero(code), "{code}");
670        }
671        for number in ["0", "0.5", "-0.5", "0e3", "10", "", "-"] {
672            assert!(!has_leading_zero(number), "{number}");
673        }
674    }
675
676    #[test]
677    fn a_change_is_the_spec_entry_in_json_and_toml() {
678        let typed = ColumnChange {
679            name: "zip".into(),
680            change: Change::Typed(ColumnType::named("date", Some("%d/%m/%Y".into())).unwrap()),
681        };
682        let json = serde_json::to_string(&typed).unwrap();
683        assert_eq!(json, r#"{"name":"zip","type":"date","format":"%d/%m/%Y"}"#);
684        assert_eq!(serde_json::from_str::<ColumnChange>(&json).unwrap(), typed);
685        assert_eq!(
686            typed.to_toml(),
687            r#"zip = { type = "date", format = "%d/%m/%Y" }"#
688        );
689        let made = ColumnChange {
690            name: "when".into(),
691            change: Change::Made {
692                from: vec!["Lcl Date".into(), "Lcl Time".into()],
693                kind: "datetime".into(),
694                format: None,
695            },
696        };
697        let json = serde_json::to_string(&made).unwrap();
698        assert_eq!(serde_json::from_str::<ColumnChange>(&json).unwrap(), made);
699        assert_eq!(
700            made.to_toml(),
701            r#"when = { from = ["Lcl Date", "Lcl Time"], as = "datetime" }"#
702        );
703        assert!(serde_json::from_str::<ColumnChange>(r#"{"name":"x","type":"int"}"#).is_err());
704    }
705
706    #[test]
707    fn every_name_is_a_type_and_back() {
708        for name in TYPE_NAMES {
709            let ty = ColumnType::named(name, None).unwrap();
710            assert_eq!(ty.name(), name);
711        }
712        let err = ColumnType::named("int", None).unwrap_err();
713        assert!(err.contains("expected one of str, bool"), "{err}");
714        assert!(ColumnType::named("i64", Some("%Y".into())).is_err());
715    }
716
717    #[test]
718    fn it_reads_and_writes_the_spec_inline_table() {
719        let ty = ColumnType::named("date", Some("%d/%m/%Y".into())).unwrap();
720        let text = ty.to_toml();
721        assert_eq!(text, r#"{ type = "date", format = "%d/%m/%Y" }"#);
722        assert_eq!(ColumnType::from_toml(&text).unwrap(), ty);
723        assert_eq!(
724            ColumnType::from_toml(r#"{ type = "i64" }"#)
725                .unwrap()
726                .to_toml(),
727            r#"{ type = "i64" }"#
728        );
729        assert!(ColumnType::from_toml(r#"{ type = "i64", unit = "x" }"#).is_err());
730    }
731
732    #[test]
733    fn integers_are_trimmed_and_unfit_values_are_counted() {
734        let (s, unfit) = typed(
735            "i64",
736            None,
737            &[Some(" 12 "), Some("x"), Some("1.5"), None, Some("  ")],
738        );
739        assert_eq!(s.dtype(), &DataType::Int64);
740        let v: Vec<Option<i64>> = s.i64().unwrap().iter().collect();
741        assert_eq!(v, [Some(12), None, None, None, None]);
742        assert_eq!(unfit[0].not_parsed, 2);
743        assert_eq!(unfit[0].out_of_range, 0);
744    }
745
746    #[test]
747    fn a_narrow_integer_counts_what_is_out_of_its_range() {
748        let (s, unfit) = typed(
749            "u8",
750            None,
751            &[Some("255"), Some("300"), Some("-1"), Some("a")],
752        );
753        let v: Vec<Option<u8>> = s.u8().unwrap().iter().collect();
754        assert_eq!(v, [Some(255), None, None, None]);
755        assert_eq!((unfit[0].not_parsed, unfit[0].out_of_range), (1, 2));
756        let notes = unfit_notes(&unfit, "");
757        assert_eq!(
758            notes[0].summary,
759            "v: 1 value not u8, 2 values out of range for u8, read as null"
760        );
761        let (s, unfit) = typed("i8", None, &[Some("127"), Some("128")]);
762        assert_eq!(s.null_count(), 1);
763        assert_eq!(unfit[0].out_of_range, 1);
764    }
765
766    #[test]
767    fn floats_and_f32_round_trip() {
768        let (s, unfit) = typed("f32", None, &[Some("1.5"), Some(" -2.25"), Some("1e3")]);
769        let v: Vec<Option<f32>> = s.f32().unwrap().iter().collect();
770        assert_eq!(v, [Some(1.5), Some(-2.25), Some(1000.0)]);
771        assert!(unfit.is_empty());
772        let (s, _) = typed("f64", None, &[Some("25.1")]);
773        assert_eq!(s.f64().unwrap().get(0), Some(25.1));
774    }
775
776    #[test]
777    fn bool_takes_words_and_digits() {
778        let (s, unfit) = typed(
779            "bool",
780            None,
781            &[
782                Some("1"),
783                Some(" 0"),
784                Some("TRUE"),
785                Some("false"),
786                Some("yes"),
787            ],
788        );
789        let v: Vec<Option<bool>> = s.bool().unwrap().iter().collect();
790        assert_eq!(v, [Some(true), Some(false), Some(true), Some(false), None]);
791        assert_eq!(unfit[0].not_parsed, 1);
792    }
793
794    #[test]
795    fn temporal_types_take_a_format() {
796        let (s, unfit) = typed(
797            "date",
798            Some("%d/%m/%Y"),
799            &[Some("03/04/2024"), Some("2024-04-03")],
800        );
801        assert_eq!(s.dtype(), &DataType::Date);
802        assert_eq!(s.get(0).unwrap().to_string(), "2024-04-03");
803        assert_eq!(unfit[0].not_parsed, 1);
804        let (s, _) = typed("time", Some("%H:%M:%S"), &[Some("13:35:42")]);
805        assert_eq!(s.dtype(), &DataType::Time);
806        let (s, _) = typed("datetime", None, &[Some("2024-03-01 10:00:00")]);
807        assert_eq!(s.get(0).unwrap().to_string(), "2024-03-01 10:00:00");
808        let (s, _) = typed("duration", None, &[Some("2h30m"), Some("x")]);
809        assert_eq!(s.null_count(), 1);
810        let (s, _) = typed("str", None, &[Some(" a ")]);
811        assert_eq!(
812            s.str().unwrap().get(0),
813            Some(" a "),
814            "text is left as it is"
815        );
816    }
817}
datui_lib/column_types.rs

datui_lib/
column_types.rs
