deser_csv/
lib.rs

1//! CSV, TSV and other delimited text for deser.
2//!
3//! ```rust
4//! use deser::{Deserialize, Serialize};
5//!
6//! #[derive(Debug, Deserialize, Serialize)]
7//! struct City {
8//!     name: String,
9//!     country: String,
10//!     population: Option<u64>,
11//! }
12//!
13//! let input =
14//!     "name,country,population\nVienna,Austria,1897000\nAtlantis,,\n";
15//! let cities: Vec<City> = deser_csv::from_str(input).unwrap();
16//! assert_eq!(cities[0].population, Some(1897000));
17//! assert_eq!(cities[1].population, None);
18//!
19//! assert_eq!(deser_csv::to_string(&cities).unwrap(), input);
20//! ```
21//!
22//! # Data Model
23//!
24//! A document is a sequence of records.  By default the first record holds
25//! the names of the columns (see [`Headers`]) and the other records are
26//! maps of the names to their fields, without names records are sequences
27//! (for instance tuples):
28//!
29//! | input                  | deser                                     |
30//! |------------------------|-------------------------------------------|
31//! | `a,b\n1,2\n3,4\n`      | `[{"a": "1", "b": "2"}, {"a": "3", "b": "4"}]` |
32//! | `1,2\n3,4\n` (no names)| `[["1", "2"], ["3", "4"]]`                |
33//!
34//! Everything in a CSV file is text, only the type a field is deserialized
35//! into knows what it means.  Fields are therefore passed on as [lexical
36//! atoms](deser_core::Atom::Lexical) which are parsed by the types they are
37//! delivered to: numbers parse them, strings take them as they are.  They
38//! are interpreted with the [lenient
39//! rules](deser_core::de::LexicalRules::LENIENT): `yes`, `on` and `1` are
40//! booleans too and empty fields are `None` for optional numbers (other
41//! [`LexicalRules`](deser_core::de::LexicalRules) can be given in the
42//! [`Context`](deser_core::Context)).  This also works when values are buffered, so flattened structs and
43//! internally tagged and untagged enums work:
44//!
45//! ```rust
46//! use deser::Deserialize;
47//!
48//! #[derive(Debug, Deserialize, PartialEq)]
49//! #[deser(tag = "kind", rename_all = "lowercase")]
50//! enum Shape {
51//!     Circle { radius: f64 },
52//!     Rect { width: f64, height: f64 },
53//! }
54//!
55//! #[derive(Debug, Deserialize, PartialEq)]
56//! struct Row {
57//!     id: u32,
58//!     #[deser(flatten)]
59//!     shape: Shape,
60//! }
61//!
62//! let input = "id,kind,radius,width,height\n1,circle,2,,\n2,rect,,3,4\n";
63//! let config =
64//!     deser_csv::DeserializerConfig::builder().nulls(deser_csv::Nulls::Empty).build();
65//! let rows: Vec<Row> = config.from_str(input).unwrap();
66//! assert_eq!(rows[1].shape, Shape::Rect { width: 3.0, height: 4.0 });
67//! ```
68//!
69//! Empty fields are `None` for optionals if the type does not accept them
70//! (an empty field is `None` for an `Option<u32>` and `Some("")` for an
71//! `Option<String>`).  Which fields are null can be configured (see
72//! [`Nulls`]).  Fields cannot hold maps or sequences, but a list in a
73//! field can be read with the [`Separated`](deser_core::adapters::Separated)
74//! adapter (`#[deser(as = Separated<';'>)]` reads `a;b;c`).
75//!
76//! Serializing works the other way around (see [`SerializerConfig`]): the
77//! value is a sequence of records which are maps (the keys of the first
78//! record are the names of the columns) or sequences.
79//!
80//! # Dialects
81//!
82//! There is no single CSV format, the configurations can be adjusted to
83//! what the other side writes and reads:
84//!
85//! * The delimiter (`,`, `;`, `\t`, `|`, the ASCII unit separator, ...),
86//!   the quote character (or no quotes at all) and if quotes are doubled
87//!   or escaped (see [`Escape`]).
88//! * The line ending (see [`Terminator`]), by default `\n`, `\r\n` and
89//!   `\r` all end records and `\n` is written.
90//! * Comment lines, blank lines, whitespace around fields (see [`Trim`]),
91//!   records with a different number of fields (`flexible`) and fields
92//!   with quotes that do not follow the rules (`lenient_quotes`).
93//! * Tab separated values as written by databases (with backslash escapes
94//!   and `\N` for null), see [`DeserializerConfig::tsv`].
95//!
96//! Input is UTF-8, a byte order mark at the start is skipped (and UTF-16
97//! input is reported as such).  Fields which are not UTF-8 are passed on as
98//! bytes.  The `sep=;` line Excel writes can be enabled with
99//! [`DeserializerConfig::set_sep_line`].
100//!
101//! # Errors
102//!
103//! Errors point at the position in the input and (with `deser-path`) at
104//! the path of the value, for instance `[3].price`.  By default quotes
105//! that do not follow the rules and records with the wrong number of fields
106//! are errors.
107//!
108//! # Streams
109//!
110//! A stream of records is read with a reader of `deser::io` which the
111//! configuration creates ([`DeserializerConfig::reader`]): every read
112//! returns the next record.  Errors of a record (like a field that does
113//! not fit the type) only discard the record.  The names of the columns
114//! are known to the stream deserializer (see
115//! [`StreamDeserializer::headers`]).
116//!
117//! ```rust
118//! # #[cfg(feature = "io")] {
119//! use deser_csv::DeserializerConfig;
120//!
121//! #[derive(deser::Deserialize)]
122//! struct Row {
123//!     name: String,
124//!     age: u32,
125//! }
126//!
127//! let input = &b"name,age\njane,42\njohn,x\nmax,7\n"[..];
128//! let mut reader = DeserializerConfig::new().reader(input);
129//! let mut ages = Vec::new();
130//! let mut errors = Vec::new();
131//! loop {
132//!     match reader.read::<Row>() {
133//!         Ok(Some(row)) => ages.push(row.age),
134//!         Ok(None) => break,
135//!         Err(err) => errors.push(err.line()),
136//!     }
137//! }
138//! assert_eq!(ages, [42, 7]);
139//! assert_eq!(errors, [Some(3)]);
140//! assert_eq!(reader.deserializer().headers().unwrap(), ["name", "age"]);
141//! # }
142//! ```
143//!
144//! Likewise every value written with a writer created by
145//! [`SerializerConfig::writer`] is a record, the names are written before
146//! the first one.  The functions that read and write a single value
147//! ([`from_reader`] and [`to_writer`]) read and write all records as a
148//! sequence, like [`from_str`] and [`to_string`].
149//!
150//! The stream serializer ([`Serializer`]) and the stream deserializer
151//! ([`StreamDeserializer`]) do not do IO themselves (see
152//! [`deser::stream`](deser_core::stream)), they also work with other kinds
153//! of IO and without the standard library.
154//!
155//! # Features
156//!
157//! * `io` (enabled by default): reading and writing streams of the
158//!   standard library, see [streams](#streams).  Requires `std`.
159//! * `speedups` (enabled by default): has no effect yet, it exists so
160//!   that all formats have it.
161//! * `std` (enabled by default): uses the standard library.  Without it
162//!   this crate only needs `alloc` (see [`no_std`](https://docs.rs/deser/latest/deser/#no_std)).
163#![doc(html_logo_url = "https://raw.githubusercontent.com/mitsuhiko/deser/main/artwork/logo.svg")]
164#![cfg_attr(not(any(feature = "std", test)), no_std)]
165
166extern crate alloc;
167
168mod de;
169mod parser;
170mod ser;
171mod stream;
172
173pub use self::de::{Deserializer, DeserializerConfig, DeserializerConfigBuilder, Records};
174#[cfg(feature = "io")]
175pub use self::ser::to_writer;
176pub use self::ser::{Serializer, SerializerConfig, SerializerConfigBuilder, to_string};
177pub use self::stream::StreamDeserializer;
178#[cfg(feature = "io")]
179pub use self::stream::from_reader;
180
181use deser_core::Error;
182use deser_core::de::Deserialize;
183
184/// Where the names of the columns come from.
185///
186/// ```
187/// use std::collections::BTreeMap;
188/// use deser_csv::{DeserializerConfig, Headers};
189///
190/// let rows: Vec<BTreeMap<String, u32>> =
191///     deser_csv::from_str("a,b\n1,2\n").unwrap();
192/// assert_eq!(rows[0]["b"], 2);
193///
194/// let config = DeserializerConfig::builder().headers(Headers::None).build();
195/// let rows: Vec<Vec<u32>> = config.from_str("1,2\n3,4\n").unwrap();
196/// assert_eq!(rows, [[1, 2], [3, 4]]);
197///
198/// let config = DeserializerConfig::builder().headers(Headers::Skip).build();
199/// let rows: Vec<(String, u32)> =
200///     config.from_str("name,age\njane,42\n").unwrap();
201/// assert_eq!(rows, [("jane".to_string(), 42)]);
202///
203/// let config =
204///     DeserializerConfig::builder().headers(Headers::Given(&["a", "b"])).build();
205/// let rows: Vec<BTreeMap<String, u32>> = config.from_str("1,2\n").unwrap();
206/// assert_eq!(rows[0]["a"], 1);
207/// ```
208#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
209#[non_exhaustive]
210pub enum Headers {
211    /// The first record holds the names, records are maps.
212    #[default]
213    First,
214    /// The first record holds the names, but records are sequences (for
215    /// instance to read them into tuples).
216    ///
217    /// The names are still read (see [`StreamDeserializer::headers`]).
218    Skip,
219    /// There are no names, records are sequences.
220    None,
221    /// The names are given (the first record is a regular record), records
222    /// are maps.
223    Given(&'static [&'static str]),
224}
225
226/// What ends records.
227///
228/// Line breaks in quoted fields do not end records.
229#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
230#[non_exhaustive]
231pub enum Terminator {
232    /// Records end with `\n`, `\r\n` or `\r` (also mixed).  `\n` is
233    /// written.
234    #[default]
235    Newline,
236    /// Like [`Newline`](Self::Newline) but `\r\n` is written (as described
237    /// by RFC 4180).
238    CrLf,
239    /// Records end with the given character (for instance the ASCII record
240    /// separator `0x1e`).
241    Byte(u8),
242}
243
244/// How characters are escaped.
245///
246/// ```
247/// use deser_csv::{DeserializerConfig, Escape, Headers};
248///
249/// let config = DeserializerConfig::builder()
250///     .headers(Headers::None)
251///     .escape(Escape::Backslash)
252///     .double_quote(false).build();
253/// let rows: Vec<Vec<String>> = config.from_str(r#""a\"b",c\,d"#).unwrap();
254/// assert_eq!(rows, [["a\"b", "c,d"]]);
255/// ```
256#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
257#[non_exhaustive]
258pub enum Escape {
259    /// Nothing is escaped (quotes in quoted fields are doubled).
260    #[default]
261    None,
262    /// The character after this one is taken as it is (in quoted and
263    /// unquoted fields), like `escapechar` of Python's `csv` module.
264    Char(u8),
265    /// Backslash escapes: `\t`, `\n`, `\r` and `\0` are a tab, line feed,
266    /// carriage return and NUL, a backslash followed by another character
267    /// is that character (`\\`, `\"`, `\,`).  This is how databases (like
268    /// PostgreSQL and MySQL) write delimited text.
269    Backslash,
270}
271
272impl Escape {
273    /// Returns the escape character.
274    pub(crate) const fn byte(self) -> Option<u8> {
275        match self {
276            Escape::None => None,
277            Escape::Char(byte) => Some(byte),
278            Escape::Backslash => Some(b'\\'),
279        }
280    }
281}
282
283/// Which whitespace is removed.
284///
285/// Spaces and tabs are removed from the start and end of unquoted fields
286/// and around the quotes of quoted fields (`a, "b" ,c`), whitespace in
287/// quotes is kept.
288///
289/// ```
290/// use deser_csv::{DeserializerConfig, Trim};
291///
292/// #[derive(deser::Deserialize)]
293/// struct Row {
294///     name: String,
295///     age: u32,
296/// }
297///
298/// let config = DeserializerConfig::builder().trim(Trim::All).build();
299/// let rows: Vec<Row> =
300///     config.from_str("name , age\n \"jane \" , 42\n").unwrap();
301/// assert_eq!((rows[0].name.as_str(), rows[0].age), ("jane ", 42));
302/// ```
303#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
304#[non_exhaustive]
305pub enum Trim {
306    /// Whitespace is kept.
307    #[default]
308    None,
309    /// Whitespace is removed from the names of the columns.
310    Headers,
311    /// Whitespace is removed from the fields of records.
312    Fields,
313    /// Whitespace is removed from names and fields.
314    All,
315}
316
317/// Which fields are null.
318///
319/// Only unquoted fields are null, which allows writing the empty string
320/// (or the text of null) in quotes.
321///
322/// ```
323/// use deser_csv::{DeserializerConfig, Headers, Nulls};
324///
325/// let mut config = DeserializerConfig::builder().headers(Headers::None).build();
326/// let rows: Vec<Vec<Option<String>>> = config.from_str(",\"\"\n").unwrap();
327/// assert_eq!(rows, [[Some("".to_string()), Some("".to_string())]]);
328///
329/// config.set_nulls(Nulls::Empty);
330/// let rows: Vec<Vec<Option<String>>> = config.from_str(",\"\"\n").unwrap();
331/// assert_eq!(rows, [[None, Some("".to_string())]]);
332/// ```
333#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
334#[non_exhaustive]
335pub enum Nulls {
336    /// No field is null.  Empty fields are still `None` for optionals of
337    /// types that do not accept the empty string.
338    #[default]
339    None,
340    /// Empty fields are null (like PostgreSQL's CSV format).
341    Empty,
342    /// Fields with this text are null (like `\N` or `NULL`).
343    Text(&'static str),
344}
345
346/// When fields are quoted.
347///
348/// ```
349/// use deser_csv::{QuoteStyle, SerializerConfig};
350///
351/// let rows = vec![("a b", 1), ("c,d", 2)];
352/// let with = |style| {
353///     SerializerConfig::builder().quote_style(style).build().to_string(&rows).unwrap()
354/// };
355/// assert_eq!(with(QuoteStyle::Necessary), "a b,1\n\"c,d\",2\n");
356/// assert_eq!(with(QuoteStyle::Always), "\"a b\",\"1\"\n\"c,d\",\"2\"\n");
357/// assert_eq!(with(QuoteStyle::NonNumeric), "\"a b\",1\n\"c,d\",2\n");
358/// ```
359#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
360#[non_exhaustive]
361pub enum QuoteStyle {
362    /// Fields are quoted if they have to be (if they contain special
363    /// characters or would otherwise be read differently).
364    #[default]
365    Necessary,
366    /// All fields are quoted.
367    Always,
368    /// All fields except for numbers are quoted.
369    NonNumeric,
370    /// Fields are never quoted, fields that need quotes are an error
371    /// (unless they can be escaped, see [`Escape`]).
372    Never,
373}
374
375/// Deserializes the records of a string.
376///
377/// This uses the default [`DeserializerConfig`] (CSV with a header), see
378/// the [crate documentation](crate) for how records map onto deser.
379///
380/// ```
381/// #[derive(deser::Deserialize)]
382/// struct Row {
383///     name: String,
384///     score: f64,
385/// }
386///
387/// let rows: Vec<Row> =
388///     deser_csv::from_str("name,score\na,1.5\nb,2\n").unwrap();
389/// assert_eq!(rows[1].score, 2.0);
390/// ```
391pub fn from_str<'de, T: Deserialize<'de>>(s: &'de str) -> Result<T, Error> {
392    DeserializerConfig::new().from_str(s)
393}
394
395/// Deserializes the records of a byte slice.
396///
397/// Fields which are not UTF-8 are passed on as bytes.
398///
399/// ```
400/// use std::collections::BTreeMap;
401///
402/// let rows: Vec<BTreeMap<String, u32>> =
403///     deser_csv::from_slice(b"a,b\n1,2\n").unwrap();
404/// assert_eq!(rows[0]["b"], 2);
405/// ```
406pub fn from_slice<'de, T: Deserialize<'de>>(bytes: &'de [u8]) -> Result<T, Error> {
407    DeserializerConfig::new().from_slice(bytes)
408}
deser_csv/lib.rs

deser_csv/
lib.rs
