mariadb-cli 0.1.1

One-shot agent-native command line client for MariaDB servers
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
//! Turning the bytes of a separated file into rows a table could take.
//!
//! Everything a file can be judged on without a server lives here: whether a
//! line can be read at all, and whether it carries the number of fields the
//! header names. Whether a value fits the column it is aimed at is a question
//! only the server can answer, and [`super::import`] asks it later.
//!
//! Splitting it this way also means the whole of this file is provable without
//! a server, which is where the load's real edge cases are: the resumed run,
//! the short line, the last partial batch.

use super::import::ImportRequest;
use crate::error::{Failure, Result};
use std::path::Path;

/// One row the load would not take, and why.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Rejection {
    /// Which row of the file it was, counting from zero past any header line.
    pub row: u64,
    /// Why it was refused, in prose the caller can act on.
    pub reason: String,
}

/// One batch of rows ready to be sent and committed together.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Batch {
    /// The offset of the first row of this batch.
    pub start: u64,
    /// The offset just past the last row of this batch.
    pub end: u64,
    /// The rows themselves, each already the width of the table.
    pub rows: Vec<Vec<String>>,
    /// Where in the file each row of `rows` came from.
    ///
    /// # Why this is not `start..end`
    ///
    /// A line the reader refuses — too few fields for the header — never
    /// reaches `rows`, while `start` and `end` count every line of the file. So
    /// the span and the rows disagree exactly when a batch has a hole in it,
    /// and anything deriving a position by adding an index to `start` names the
    /// wrong line from the hole onwards.
    ///
    /// An audit measured both ways of getting that wrong: walking the span
    /// invented a refusal for a line that was never sent, and walking the rows
    /// with `start + index` named a line that had been written while staying
    /// silent about the one that failed. Carrying the position with the row is
    /// what makes either impossible.
    pub positions: Vec<u64>,
}

/// What one pass over the file produced.
#[derive(Debug, Clone, Default)]
pub struct Reading {
    /// The column names, when the file named them.
    pub columns: Vec<String>,
    /// The batches, in the order they must be sent.
    pub batches: Vec<Batch>,
    /// Rows the file itself could not offer, with the reason for each.
    pub rejected: Vec<Rejection>,
    /// How many rows were read, refusals included.
    pub rows_read: u64,
}

/// A file being turned into batches, one batch at a time.
///
/// # Why the file is not read into memory first
///
/// It used to be. Every batch of the file was built and kept, and only then was
/// the first one sent, so a load held the whole file in memory at once: two
/// gigabytes of separated text arrived as rather more than two gigabytes of
/// `String`, since every field carries its own header besides its content. The
/// module said "one committed batch at a time" and meant it; the reading did
/// not do it.
///
/// Held here instead is one batch, which is what `batch_size` was always meant
/// to bound. A file of any size loads under the same ceiling, and the ceiling
/// is a number the caller sets rather than a property of the file they happened
/// to pick.
pub struct Batches {
    /// The file, read one record at a time.
    reader: csv::Reader<std::fs::File>,
    /// How many rows one batch carries.
    batch_size: usize,
    /// The row this run starts at, so a resumed load skips what already went in.
    resume_from: u64,
    /// How wide a line has to be, when a header named the columns.
    width: usize,
    /// The names the file gave its columns, empty when it named none.
    columns: Vec<String>,
    /// Where in the file the walk is, counting every line including skipped.
    offset: u64,
    /// Where the next batch begins.
    start: u64,
    /// Rows refused since the caller last collected them.
    rejected: Vec<Rejection>,
    /// How many rows were read, refusals included.
    rows_read: u64,
}

impl Batches {
    /// Opens the file and reads its header, if it has one.
    ///
    /// # Errors
    ///
    /// Returns a configuration failure when the file cannot be opened, and a
    /// usage failure when a batch of no rows was asked for, which would never
    /// commit.
    pub fn open(file: &Path, request: &ImportRequest) -> Result<Self> {
        if request.batch_size == 0 {
            return Err(Failure::usage(
                "datamove.empty_batch",
                "a batch of no rows would never commit anything",
            )
            .with_hint("ask for at least one row per batch"));
        }

        let mut reader = csv::ReaderBuilder::new()
            .delimiter(request.separator)
            .has_headers(request.header)
            .flexible(true)
            .from_path(file)
            .map_err(|error| unreadable(file, &error))?;

        let columns: Vec<String> = if request.header {
            let named = reader.headers().map_err(|error| unreadable(file, &error))?;
            named.iter().map(ToOwned::to_owned).collect()
        } else {
            Vec::new()
        };

        Ok(Self {
            reader,
            batch_size: request.batch_size,
            resume_from: request.resume_from,
            width: columns.len(),
            columns,
            offset: 0,
            start: request.resume_from,
            rejected: Vec::new(),
            rows_read: 0,
        })
    }

    /// The names the file gave its columns, empty when it named none.
    #[must_use]
    pub fn columns(&self) -> &[String] {
        &self.columns
    }

    /// How many rows have been read so far, refusals included.
    #[must_use]
    pub const fn rows_read(&self) -> u64 {
        self.rows_read
    }

    /// The rows refused since this was last called, and forgets them.
    ///
    /// Handed over rather than kept, because a load of a badly formed file
    /// would otherwise grow a list as long as the file itself, which is the
    /// same unbounded holding this type exists to stop.
    pub fn take_rejected(&mut self) -> Vec<Rejection> {
        std::mem::take(&mut self.rejected)
    }

    /// The next batch, or `None` once the file is spent.
    ///
    /// A batch short of `batch_size` is only ever the last one, and a file whose
    /// remaining lines are all refused ends here without a final empty batch.
    pub fn next_batch(&mut self) -> Option<Batch> {
        let mut rows = Vec::with_capacity(self.batch_size);
        let mut positions = Vec::with_capacity(self.batch_size);

        while let Some(record) = self.reader.records().next() {
            let position = self.offset;
            self.offset += 1;
            if position < self.resume_from {
                continue;
            }
            self.rows_read += 1;

            match fields_of(record, self.width) {
                Ok(fields) => {
                    rows.push(fields);
                    positions.push(position);
                }
                Err(reason) => {
                    self.rejected.push(Rejection {
                        row: position,
                        reason,
                    });
                    continue;
                }
            }

            if rows.len() >= self.batch_size {
                let end = position + 1;
                let start = self.start;
                self.start = end;
                return Some(Batch {
                    start,
                    end,
                    rows,
                    positions,
                });
            }
        }

        if rows.is_empty() {
            return None;
        }
        let start = self.start;
        self.start = self.offset;
        Some(Batch {
            start,
            end: self.offset,
            rows,
            positions,
        })
    }
}

/// Reads the whole file into batches at once, holding all of them.
///
/// Used by the proofs, which measure how a file is cut up and read more clearly
/// against the finished list than against a walk. THE LOAD DOES NOT USE THIS:
/// it drives [`Batches`] directly, because holding every batch is exactly what
/// a load of a file larger than memory must not do. Anything reaching for this
/// on a real file is choosing to hold it whole and should say why.
///
/// # Errors
///
/// Returns a configuration failure when the file cannot be opened, and a usage
/// failure when a batch of no rows was asked for, which would never commit.
pub fn read(file: &Path, request: &ImportRequest) -> Result<Reading> {
    let mut batches = Batches::open(file, request)?;
    let mut reading = Reading::default();
    while let Some(batch) = batches.next_batch() {
        reading.batches.push(batch);
        reading.rejected.extend(batches.take_rejected());
    }
    reading.rejected.extend(batches.take_rejected());
    reading.rows_read = batches.rows_read();
    reading.columns = batches.columns().to_vec();
    Ok(reading)
}

/// One line as fields, or the reason it cannot be offered to the table.
///
/// The reason is prose rather than a failure because a refused line does not
/// end the load: it is carried alongside the rows that did go in.
fn fields_of(
    record: csv::Result<csv::StringRecord>,
    width: usize,
) -> std::result::Result<Vec<String>, String> {
    let record = record.map_err(|error| format!("the line could not be read: {error}"))?;
    if width > 0 && record.len() != width {
        return Err(format!(
            "the line carries {} fields where the header names {width}",
            record.len()
        ));
    }
    Ok(record.iter().map(ToOwned::to_owned).collect())
}

/// The failure an unreadable file produces.
///
/// The hint names who opened the file, because a caller looking at a path
/// failure on a database tool will otherwise wonder whether the server was
/// meant to find it.
fn unreadable(file: &Path, cause: &impl std::fmt::Display) -> Failure {
    Failure::config(
        "datamove.file_unreadable",
        format!("`{}` could not be read", file.display()),
    )
    .with_detail("detail", file.display().to_string())
    .with_hint("this process opens the file, so the path is read from here and not from the server")
    .with_cause(cause.to_string())
}

#[cfg(test)]
mod tests {
    use super::*;
    use std::path::PathBuf;

    fn request(file: &Path, batch_size: usize) -> ImportRequest {
        ImportRequest {
            file: file.to_path_buf(),
            table: "customers".to_owned(),
            database: None,
            batch_size,
            resume_from: 0,
            separator: b',',
            header: true,
            stop_on_refusal: false,
        }
    }

    fn write_file(content: &str) -> (tempfile::TempDir, PathBuf) {
        let directory = tempfile::tempdir().expect("scratch directory");
        let path = directory.path().join("rows.csv");
        std::fs::write(&path, content).expect("writes");
        (directory, path)
    }

    /// A long file is walked without ever holding more than one batch.
    ///
    /// The number that matters is what is held at once, and the way to measure
    /// it without a memory profiler is to ask the walk what it is carrying: at
    /// no point may a batch be wider than the size asked for, and no round may
    /// leave the previous batch behind still owned.
    ///
    /// This is the shape the module always described and did not have. Reading
    /// built every batch before sending the first, so a two gigabyte file was
    /// two gigabytes resident — rather more, since every field carried its own
    /// header besides its content. The ceiling was written as `batch_size` and
    /// was in truth the size of whatever file the caller happened to pick.
    #[test]
    fn a_long_file_is_walked_without_holding_more_than_one_batch() {
        use std::fmt::Write as _;

        let mut content = String::from("id,name\n");
        for row in 0..5_000 {
            writeln!(content, "{row},name-{row}").expect("a string always takes a write");
        }
        let (_guard, path) = write_file(&content);

        let mut batches = Batches::open(&path, &request(&path, 100)).expect("opens");
        let mut seen = 0_u64;
        let mut rounds = 0_usize;
        while let Some(batch) = batches.next_batch() {
            assert!(
                batch.rows.len() <= 100,
                "a batch carried {} rows against a ceiling of 100",
                batch.rows.len()
            );
            seen += batch.rows.len() as u64;
            rounds += 1;
            // The batch just measured is dropped here, at the end of the round.
            // If the walk were holding them all, this would be the round that
            // failed to reclaim anything, and the assertion below on the count
            // of rounds would still pass — which is why the width above is
            // asserted every round rather than only at the end.
        }

        assert_eq!(seen, 5_000, "every row was walked");
        assert_eq!(rounds, 50, "five thousand rows in batches of a hundred");
        assert_eq!(batches.rows_read(), 5_000);
    }

    /// Each batch also publishes the offset a later run continues from, which
    /// is the only thing that makes an interrupted load resumable.
    #[test]
    fn the_rows_are_cut_into_batches_that_each_carry_their_offset() {
        let (_guard, path) = write_file("id\n1\n2\n3\n4\n5\n");
        let reading = read(&path, &request(&path, 2)).expect("reads");
        assert_eq!(reading.batches.len(), 3);
        assert_eq!(reading.batches[0].start, 0);
        assert_eq!(reading.batches[2].rows.len(), 1);
        let ends: Vec<u64> = reading.batches.iter().map(|batch| batch.end).collect();
        assert_eq!(ends, vec![2, 4, 5]);
    }

    #[test]
    fn a_resumed_load_starts_where_the_last_one_stopped() {
        let (_guard, path) = write_file("id\n1\n2\n3\n4\n");
        let mut asked = request(&path, 10);
        asked.resume_from = 2;

        let reading = read(&path, &asked).expect("reads");
        assert_eq!(reading.rows_read, 2, "the rows already loaded are skipped");
        assert_eq!(reading.batches.len(), 1);
        assert_eq!(reading.batches[0].start, 2);
        assert_eq!(reading.batches[0].end, 4);
        assert_eq!(
            reading.batches[0].rows,
            vec![vec!["3".to_owned()], vec!["4".to_owned()]]
        );
    }

    /// Each refused row is named with its own position and its own reason,
    /// because a total tells the caller nothing they can act on, and the rows
    /// that do fit are still loaded.
    #[test]
    fn every_refused_row_is_named_and_the_rest_still_load() {
        let (_guard, path) = write_file("id,name\n1\n2,bruno\n3\n4,dora\n");
        let reading = read(&path, &request(&path, 10)).expect("reads");

        let positions: Vec<u64> = reading.rejected.iter().map(|entry| entry.row).collect();
        assert_eq!(positions, vec![0, 2]);
        for entry in &reading.rejected {
            assert!(entry.reason.contains("fields"), "a reason, not a count");
        }
        assert_eq!(reading.batches[0].rows.len(), 2, "the rest still load");
    }

    #[test]
    fn a_file_with_no_header_takes_every_line_as_a_row() {
        let (_guard, path) = write_file("1,ana\n2,bruno\n");
        let mut asked = request(&path, 10);
        asked.header = false;

        let reading = read(&path, &asked).expect("reads");
        assert!(reading.columns.is_empty());
        assert_eq!(reading.rows_read, 2);
    }

    /// Whatever separates the fields, the header still names the columns and
    /// the rows still arrive whole.
    #[test]
    fn another_separator_is_read_the_same_way() {
        for (content, separator) in [("id,name\n1,ana\n", b','), ("id\tname\n1\tana\n", b'\t')] {
            let (_guard, path) = write_file(content);
            let mut asked = request(&path, 10);
            asked.separator = separator;

            let reading = read(&path, &asked).expect("reads");
            assert_eq!(reading.columns, vec!["id".to_owned(), "name".to_owned()]);
            assert_eq!(
                reading.batches[0].rows[0],
                vec!["1".to_owned(), "ana".to_owned()]
            );
            assert!(reading.rejected.is_empty());
        }
    }

    #[test]
    fn a_file_that_is_not_there_is_a_configuration_failure_of_this_side() {
        let absent = Path::new("/nowhere/at/all/rows.csv");
        let failure = read(absent, &request(absent, 10)).expect_err("must refuse");
        assert_eq!(failure.code(), "datamove.file_unreadable");
        let hint = failure.hint().unwrap_or_default();
        assert!(hint.contains("not from the server"), "{hint}");
    }

    #[test]
    fn a_batch_of_no_rows_is_refused_rather_than_looping_forever() {
        let (_guard, path) = write_file("id\n1\n");
        let failure = read(&path, &request(&path, 0)).expect_err("must refuse");
        assert_eq!(failure.code(), "datamove.empty_batch");
        assert_eq!(failure.exit_number().code(), 2);
    }
}