probl_sema/
data.rs

1//! Reading data: the rules that the compiler, the loader and `probl schema`
2//! share (docs/data-input.md).
3
4use crate::ir::{DataFormat, EnumType, RecordType, TypeSpec};
5use probl_syntax::Span;
6
7/// The key a name is matched by, between record fields and the columns or
8/// keys of data: its letters and digits, in lowercase. `daily_sign_ups`,
9/// `Daily sign-ups` and `dailySignUps` all have the key `dailysignups`.
10pub fn field_key(name: &str) -> String {
11    name.chars()
12        .filter(|c| c.is_alphanumeric())
13        .flat_map(char::to_lowercase)
14        .collect()
15}
16
17impl DataFormat {
18    pub fn name(self) -> &'static str {
19        match self {
20            DataFormat::Csv => "csv",
21            DataFormat::Json => "json",
22            DataFormat::Lines => "lines",
23        }
24    }
25
26    pub fn from_name(name: &str) -> Option<DataFormat> {
27        match name {
28            "csv" => Some(DataFormat::Csv),
29            "json" => Some(DataFormat::Json),
30            "lines" => Some(DataFormat::Lines),
31            _ => None,
32        }
33    }
34
35    /// The format a path's extension implies. Standard input (`-`) is lines.
36    pub fn from_path(path: &str) -> Option<DataFormat> {
37        if path == "-" {
38            return Some(DataFormat::Lines);
39        }
40        let extension = path.rsplit_once('.')?.1.to_ascii_lowercase();
41        match extension.as_str() {
42            "csv" => Some(DataFormat::Csv),
43            "json" => Some(DataFormat::Json),
44            "txt" => Some(DataFormat::Lines),
45            _ => None,
46        }
47    }
48}
49
50/// A single value: a CSV cell, a line, or a JSON number, string or boolean.
51pub fn is_plain(ty: &TypeSpec) -> bool {
52    matches!(
53        ty,
54        TypeSpec::Int
55            | TypeSpec::Float
56            | TypeSpec::Prob
57            | TypeSpec::Bool
58            | TypeSpec::Str
59            | TypeSpec::Date
60            | TypeSpec::Enum(_)
61    )
62}
63
64/// What a JSON object's keys, which are text, can be read as.
65pub fn is_key(ty: &TypeSpec) -> bool {
66    matches!(
67        ty,
68        TypeSpec::Int | TypeSpec::Bool | TypeSpec::Str | TypeSpec::Date | TypeSpec::Enum(_)
69    )
70}
71
72/// Why data can't be read as a type.
73#[derive(Clone, Debug, PartialEq)]
74pub struct Problem {
75    pub message: String,
76    pub help: Option<String>,
77    /// A record field's declaration, when the problem is there.
78    pub at: Option<Span>,
79}
80
81impl Problem {
82    fn new(message: impl Into<String>, help: Option<&str>, at: Option<Span>) -> Problem {
83        Problem {
84            message: message.into(),
85            help: help.map(str::to_string),
86            at,
87        }
88    }
89}
90
91/// Check that data in `format` can be read as `ty`: that `ty` holds only
92/// values, fits the format, and has no two fields that would match the same
93/// name in the data.
94pub fn check(ty: &TypeSpec, format: DataFormat, records: &[RecordType], enums: &[EnumType]) -> Vec<Problem> {
95    let mut checker = Checker {
96        records,
97        enums,
98        seen: vec![false; records.len()],
99        problems: Vec::new(),
100    };
101    let describe = |t: &TypeSpec| t.describe_in(records, enums);
102    match format {
103        DataFormat::Json => checker.value(ty, None),
104        DataFormat::Csv => match ty {
105            TypeSpec::List(row) if matches!(**row, TypeSpec::Record(_) | TypeSpec::AnonRecord(_)) => checker.row(row),
106            _ => checker.problems.push(Problem::new(
107                format!(
108                    "a CSV file reads as a list of records, one per row, not as `{}`",
109                    describe(ty)
110                ),
111                Some("declare a record type for the rows: `type Row = { … }`, then `let rows: list[Row] = read(…)`"),
112                None,
113            )),
114        },
115        DataFormat::Lines => match ty {
116            TypeSpec::List(t) if is_plain(t) => {}
117            _ => checker.problems.push(Problem::new(
118                format!("lines read as a list of single values, not as `{}`", describe(ty)),
119                Some("read one value per line, like `list[int]` or `list[str]`"),
120                None,
121            )),
122        },
123    }
124    checker.problems
125}
126
127struct Checker<'a> {
128    records: &'a [RecordType],
129    enums: &'a [EnumType],
130    /// Named record types already checked.
131    seen: Vec<bool>,
132    problems: Vec<Problem>,
133}
134
135impl Checker<'_> {
136    fn describe(&self, ty: &TypeSpec) -> String {
137        ty.describe_in(self.records, self.enums)
138    }
139
140    /// Any JSON value.
141    fn value(&mut self, ty: &TypeSpec, at: Option<Span>) {
142        match ty {
143            _ if is_plain(ty) => {}
144            TypeSpec::List(t) | TypeSpec::Bag(t) => self.value(t, at),
145            TypeSpec::Map(k, v) => {
146                if !is_key(k) {
147                    self.problems.push(Problem::new(
148                        format!(
149                            "a map read from JSON has text keys, so they can't be `{}`",
150                            self.describe(k)
151                        ),
152                        Some("a map's keys can be `str`, `int`, `bool`, `date` or an enum"),
153                        at,
154                    ));
155                }
156                self.value(v, at);
157            }
158            TypeSpec::Record(r) => {
159                if std::mem::replace(&mut self.seen[*r as usize], true) {
160                    return;
161                }
162                let fields: Vec<(String, TypeSpec, Option<Span>)> = self.records[*r as usize]
163                    .fields
164                    .iter()
165                    .map(|f| (f.name.clone(), f.ty.clone(), Some(f.span)))
166                    .collect();
167                self.distinct_names(&fields, at);
168                for (_, t, span) in &fields {
169                    self.value(t, span.or(at));
170                }
171            }
172            TypeSpec::AnonRecord(fields) => {
173                let fields: Vec<(String, TypeSpec, Option<Span>)> =
174                    fields.iter().map(|(n, t)| (n.clone(), t.clone(), None)).collect();
175                self.distinct_names(&fields, at);
176                for (_, t, _) in &fields {
177                    self.value(t, at);
178                }
179            }
180            TypeSpec::Dist(_) => self.problems.push(Problem::new(
181                format!("data can't be a distribution, like `{}`", self.describe(ty)),
182                Some("read the values, then make a distribution of them, like `one_of(values)`"),
183                at,
184            )),
185            _ => self.problems.push(Problem::new(
186                format!("data can't be `{}`", self.describe(ty)),
187                Some("data is numbers, text, dates, enums, and records, lists, maps and bags of those"),
188                at,
189            )),
190        }
191    }
192
193    /// A CSV row: a record of single values.
194    fn row(&mut self, ty: &TypeSpec) {
195        let fields: Vec<(String, TypeSpec, Option<Span>)> = match ty {
196            TypeSpec::Record(r) => self.records[*r as usize]
197                .fields
198                .iter()
199                .map(|f| (f.name.clone(), f.ty.clone(), Some(f.span)))
200                .collect(),
201            TypeSpec::AnonRecord(fields) => fields.iter().map(|(n, t)| (n.clone(), t.clone(), None)).collect(),
202            _ => return,
203        };
204        self.distinct_names(&fields, None);
205        for (name, t, span) in &fields {
206            if !is_plain(t) {
207                self.problems.push(Problem::new(
208                    format!(
209                        "a CSV cell holds a single value, but `{name}` is a `{}`",
210                        self.describe(t)
211                    ),
212                    Some("read the file as JSON, or give each value a column of its own"),
213                    *span,
214                ));
215            }
216        }
217    }
218
219    /// No two fields may match the same name in the data.
220    fn distinct_names(&mut self, fields: &[(String, TypeSpec, Option<Span>)], at: Option<Span>) {
221        for (i, (a, _, span)) in fields.iter().enumerate() {
222            let key = field_key(a);
223            if let Some((b, _, _)) = fields[..i].iter().find(|(b, _, _)| field_key(b) == key) {
224                self.problems.push(Problem::new(
225                    format!("the fields `{b}` and `{a}` would match the same names in the data"),
226                    Some("names match ignoring case, spaces and punctuation: rename one of the fields"),
227                    span.or(at),
228                ));
229            }
230        }
231    }
232}
233
234#[cfg(test)]
235mod tests {
236    use super::*;
237
238    #[test]
239    fn names_match_ignoring_case_and_punctuation() {
240        for name in ["daily_sign_ups", "Daily sign-ups", "dailySignUps", "DAILY_SIGN_UPS"] {
241            assert_eq!(field_key(name), "dailysignups");
242        }
243        // Other alphabets and digits count too, lowercased where they can be.
244        assert_eq!(field_key("Größe (m²)"), "größem²");
245        assert_ne!(field_key("a1"), field_key("a_2"));
246    }
247
248    #[test]
249    fn formats_come_from_extensions() {
250        assert_eq!(DataFormat::from_path("data/pilot.CSV"), Some(DataFormat::Csv));
251        assert_eq!(DataFormat::from_path("a.json"), Some(DataFormat::Json));
252        assert_eq!(DataFormat::from_path("counts.txt"), Some(DataFormat::Lines));
253        assert_eq!(DataFormat::from_path("-"), Some(DataFormat::Lines));
254        assert_eq!(DataFormat::from_path("sales.xlsx"), None);
255        assert_eq!(DataFormat::from_path("README"), None);
256    }
257}
probl_sema/data.rs

probl_sema/
data.rs
