mariadb-cli 0.1.1

One-shot agent-native command line client for MariaDB servers
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
//! The same operations, applied to one arriving row at a time.

use super::value::{lookup_row, project_row, redact_in_place, scalar_text, truncate_in_place};
use super::Pipeline;
use crate::output::envelope::Row;

/// What the flow does with one arriving row.
#[derive(Debug)]
pub enum Step {
    /// Write this row, then keep pulling.
    Emit(Row),
    /// Drop this row, then keep pulling.
    Skip,
    /// Stop pulling: nothing arriving after this row belongs to the answer.
    Stop,
}

/// The output pipeline applied to one arriving row at a time.
///
/// Nothing here holds a collection of rows, so the memory cost does not grow
/// with the size of the result set. The one thing that does grow is the set of
/// keys deduplication has already seen, and only when deduplication was asked
/// for: remembering which keys appeared is what the operation is.
// Allowed: these four are independent observations a flow makes about itself
// as rows pass, and nothing reads them together. Folding them into one record
// would publish a grouping that means nothing to a caller, and folding them
// into a set of flags would trade four names a reader understands for one
// integer they would have to decode. The count is the lint's concern; what it
// is protecting against — a struct whose fields are really one state machine —
// is not what these are.
#[allow(clippy::struct_excessive_bools)]
pub struct RowFlow<'a> {
    pub(super) pipeline: &'a Pipeline,
    /// Only the keys already seen, never the rows they came from.
    pub(super) seen: std::collections::BTreeSet<String>,
    /// How many rows the resume mark has passed over so far.
    pub(super) passed_over: usize,
    /// How many rows the flow was offered, which is what the pipeline received.
    pub(super) offered: usize,
    pub(super) emitted: usize,
    /// Bytes of the volume ceiling already spent.
    ///
    /// Starts at what the answer's own frame costs rather than at zero, when
    /// the caller measured it: the ceiling covers everything that reaches the
    /// channel, and counting only rows left every flow over it by the size of
    /// its own wrapping (FR-039).
    pub(super) bytes: usize,
    pub(super) whole: bool,
    /// Whether any cell was shortened on its way past.
    pub(super) shortened: bool,
    /// Whether any cell had credential-shaped text replaced on its way past.
    pub(super) redacted: bool,
    /// Whether a row arrived that the ceiling on rows had no room for.
    pub(super) over_the_ceiling: bool,
}

impl RowFlow<'_> {
    /// Offers one arriving row to the flow.
    ///
    /// The operations run in the same fixed order the gathered path uses, so a
    /// request answered as a flow and the same request answered whole select
    /// the same rows.
    pub fn offer(&mut self, row: Row) -> Step {
        // Counted before any test, because what the pipeline received is what
        // the gathered path counts too, and a row it refused is still a row it
        // received.
        self.offered += 1;

        if !self
            .pipeline
            .filters
            .iter()
            .all(|filter| filter.accepts_row(&row))
        {
            return Step::Skip;
        }

        if let Some(path) = &self.pipeline.dedupe_by {
            // A row missing the key is always kept: absence is not a repeated
            // value, and remembering absence would grow without bound.
            if let Some(found) = lookup_row(&row, path) {
                if !self.seen.insert(scalar_text(found).into_owned()) {
                    return Step::Skip;
                }
            }
        }

        // The mark is passed over after the tests that decide which rows belong
        // to the answer, exactly as the gathered path does, so a request served
        // as a flow pages over the same rows as the same request served whole.
        if self
            .pipeline
            .cursor
            .is_some_and(|mark| self.passed_over < mark)
        {
            self.passed_over += 1;
            return Step::Skip;
        }

        // The ceiling stops the flow instead of cutting a gathered answer, so
        // the rows past it are never pulled, never converted and never held.
        if self
            .pipeline
            .limit
            .is_some_and(|ceiling| self.emitted >= ceiling)
        {
            self.over_the_ceiling = true;
            return Step::Stop;
        }

        let mut row = if self.pipeline.select.is_empty() {
            row
        } else {
            project_row(&row, &self.pipeline.select)
        };

        // Before the shortening, never after: a credential cut to fit would
        // leave the part that fitted standing in the answer.
        if self.pipeline.redact {
            for value in row.values_mut() {
                self.redacted |= redact_in_place(value);
            }
        }

        if let Some(ceiling) = self.pipeline.truncate_cell {
            for value in row.values_mut() {
                self.shortened |= truncate_in_place(value, ceiling);
            }
        }

        if let Some(ceiling) = self.pipeline.max_output_bytes {
            let size = serde_json::to_string(&row).map_or(0, |text| text.len() + 1);
            if self.bytes + size > ceiling {
                // The answer stops short of what the caller asked to read, so
                // the flow is cut and must not be allowed to look whole.
                self.whole = false;
                return Step::Stop;
            }
            self.bytes += size;
        }

        self.emitted += 1;
        Step::Emit(row)
    }

    /// Whether the flow that stopped here delivered everything it promised.
    ///
    /// A flow stopped by the row ceiling is whole, because the ceiling is what
    /// the caller asked for. A flow stopped by the volume ceiling is not: rows
    /// the caller asked to read were left behind.
    #[must_use]
    pub const fn is_whole(&self) -> bool {
        self.whole
    }

    /// How many rows the flow has passed on so far.
    #[must_use]
    pub const fn emitted(&self) -> usize {
        self.emitted
    }

    /// How many rows this flow is holding on to.
    ///
    /// Always zero. Exposed so a test can show that a long flow holds no rows
    /// rather than asserting it in prose.
    #[must_use]
    pub const fn held_rows(&self) -> usize {
        0
    }

    /// How many deduplication keys are remembered.
    ///
    /// Zero unless deduplication was asked for, so a test can show that the
    /// only state which grows is the one the operation is made of.
    #[must_use]
    pub fn tracked_keys(&self) -> usize {
        self.seen.len()
    }

    /// What the pipeline did, in the shape the gathered path reports it in.
    ///
    /// The same fields, filled from the same facts: how many rows the pipeline
    /// received, how many it passed on, how many it dropped between the two,
    /// whether any cell was shortened, and where the next page starts. A shape
    /// that reports this gathered has to report it flowing, or one request
    /// would be answered two ways depending on which path served it.
    ///
    /// Answers `None` for a pipeline that would leave every row exactly as it
    /// arrived, because the gathered path reports nothing at all for that one
    /// and an empty report is a different claim from no report.
    ///
    /// What the volume ceiling dropped is deliberately absent: a flow it cut
    /// never closes, so there is no document for it to be reported in.
    #[must_use]
    pub fn applied(&self) -> Option<crate::output::envelope::Applied> {
        if self.pipeline.is_identity() {
            return None;
        }

        let mut applied = crate::output::envelope::Applied {
            input_count: Some(self.offered),
            output_count: Some(self.emitted),
            content_truncated: self.shortened,
            content_redacted: self.redacted,
            accumulated_for: self.pipeline.accumulated_for(),
            ..crate::output::envelope::Applied::default()
        };

        let dropped = self.offered.saturating_sub(self.emitted);
        if dropped > 0 {
            applied.dropped = Some(dropped);
        }

        // A flow the volume ceiling stopped says so here, because here is the
        // only place left that can say it. The document used to carry the mark
        // in its own shape, by never closing; now that it closes, a reader
        // that does not read this field has no way to know rows were left
        // behind. The gathered path has always set this, and the two paths
        // must answer alike or the caller meets a different answer depending
        // on which one served it.
        if !self.whole {
            applied.output_truncated = true;
        }

        if self.pipeline.limit.is_some() {
            applied.has_more = Some(self.over_the_ceiling);
        } else if self.pipeline.cursor.is_some() {
            applied.has_more = Some(false);
        }
        if self.pipeline.cursor.is_some() || self.pipeline.limit.is_some() {
            applied.next_cursor = Some(self.pipeline.cursor.unwrap_or(0) + self.emitted);
        }

        Some(applied)
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::output::ops::Filter;
    use serde_json::json;

    /// One row, built the way the read path builds it.
    fn wire_row(index: i64, engine: &str) -> Row {
        let mut row = Row::new();
        row.insert("id".to_owned(), json!(index));
        row.insert("engine".to_owned(), json!(engine));
        row
    }

    /// Drives a whole flow and returns what it passed on.
    fn flowed(pipeline: &Pipeline, rows: impl Iterator<Item = Row>) -> (Vec<Row>, bool) {
        let mut stage = pipeline.flow();
        let mut kept = Vec::new();
        let mut whole = true;
        for row in rows {
            match stage.offer(row) {
                Step::Emit(shaped) => kept.push(shaped),
                Step::Skip => {}
                Step::Stop => {
                    whole = stage.is_whole();
                    break;
                }
            }
        }
        (kept, whole)
    }

    /// The two paths must select the same rows, or the same request would mean
    /// one thing gathered and another in flight.
    #[test]
    fn a_filter_selects_the_same_rows_gathered_or_in_flight() {
        let pipeline = Pipeline {
            filters: vec![Filter::parse("engine=InnoDB").expect("parses")],
            ..Pipeline::default()
        };
        let arriving = vec![
            wire_row(1, "InnoDB"),
            wire_row(2, "Aria"),
            wire_row(3, "InnoDB"),
        ];
        let (kept, _) = flowed(&pipeline, arriving.into_iter());
        assert_eq!(kept.len(), 2);
        assert_eq!(kept[0]["id"], json!(1));
        assert_eq!(kept[1]["id"], json!(3));
    }

    /// The ceiling must stop the pull rather than trim a gathered answer, or a
    /// million rows would still be read to return ten.
    #[test]
    fn the_row_ceiling_stops_the_pull_and_leaves_the_flow_whole() {
        let pipeline = Pipeline {
            limit: Some(3),
            ..Pipeline::default()
        };
        let mut stage = pipeline.flow();
        let mut kept = 0;
        let mut offered = 0;
        for index in 0..1_000_000 {
            offered += 1;
            match stage.offer(wire_row(index, "InnoDB")) {
                Step::Emit(_) => kept += 1,
                Step::Skip => {}
                Step::Stop => break,
            }
        }
        assert_eq!(kept, 3);
        assert_eq!(
            offered, 4,
            "one row past the ceiling ends the flow, no more"
        );
        assert!(stage.is_whole(), "the ceiling is what the caller asked for");
    }

    /// A projection in flight must drop the same fields a gathered one drops.
    #[test]
    fn a_projection_keeps_only_the_asked_paths() {
        let pipeline = Pipeline {
            select: vec!["id".to_owned(), "absent".to_owned()],
            ..Pipeline::default()
        };
        let (kept, _) = flowed(&pipeline, std::iter::once(wire_row(1, "InnoDB")));
        assert_eq!(kept[0].len(), 1);
        assert!(kept[0].contains_key("id"));
        assert!(!kept[0].contains_key("absent"));
    }

    /// Deduplication may remember keys, and must never remember rows.
    #[test]
    fn deduplication_carries_keys_and_never_rows() {
        let pipeline = Pipeline {
            dedupe_by: Some("engine".to_owned()),
            ..Pipeline::default()
        };
        let mut stage = pipeline.flow();
        let mut kept = 0;
        for index in 0..10_000 {
            if matches!(stage.offer(wire_row(index, "InnoDB")), Step::Emit(_)) {
                kept += 1;
            }
        }
        assert_eq!(kept, 1, "one value, one row");
        assert_eq!(
            stage.tracked_keys(),
            1,
            "one key remembered, not ten thousand"
        );
        assert_eq!(stage.held_rows(), 0);
    }

    /// A row missing the key is not a repeat, so it is kept and not remembered.
    #[test]
    fn deduplication_keeps_the_keyless_without_remembering_it() {
        let pipeline = Pipeline {
            dedupe_by: Some("absent".to_owned()),
            ..Pipeline::default()
        };
        let mut stage = pipeline.flow();
        for index in 0..100 {
            assert!(matches!(
                stage.offer(wire_row(index, "InnoDB")),
                Step::Emit(_)
            ));
        }
        assert_eq!(stage.tracked_keys(), 0);
    }

    /// The volume ceiling stops the flow, and what it stopped is not whole.
    #[test]
    fn the_volume_ceiling_stops_the_flow_and_declares_it_cut() {
        let pipeline = Pipeline {
            max_output_bytes: Some(60),
            ..Pipeline::default()
        };
        let arriving = (0..100).map(|index| wire_row(index, "InnoDB"));
        let (kept, whole) = flowed(&pipeline, arriving);
        assert!(!kept.is_empty(), "what fits is still delivered");
        assert!(kept.len() < 100);
        assert!(!whole, "rows the caller asked to read were left behind");
    }

    /// A request served as a flow must page over the same rows as the same
    /// request served whole, or a mark would mean two different things.
    #[test]
    fn the_mark_passes_over_the_same_rows_gathered_or_in_flight() {
        let pipeline = Pipeline {
            cursor: Some(2),
            limit: Some(2),
            ..Pipeline::default()
        };
        let arriving = (0..6).map(|index| wire_row(index, "InnoDB"));
        let (kept, _) = flowed(&pipeline, arriving);
        let seen: Vec<i64> = kept
            .iter()
            .map(|row| row["id"].as_i64().expect("number"))
            .collect();
        assert_eq!(seen, [2, 3], "the mark passes over the first two rows");
    }

    /// A long flow must hold no rows at any point, or the guarantee is prose.
    #[test]
    fn a_long_flow_holds_no_rows_from_the_first_to_the_last() {
        let pipeline = Pipeline {
            filters: vec![Filter::parse("engine=InnoDB").expect("parses")],
            limit: Some(1_000_000),
            select: vec!["id".to_owned()],
            truncate_cell: Some(8),
            ..Pipeline::default()
        };
        let mut stage = pipeline.flow();
        for index in 0..200_000 {
            let _ = stage.offer(wire_row(index, "InnoDB"));
            assert_eq!(stage.held_rows(), 0);
            assert_eq!(
                stage.tracked_keys(),
                0,
                "no deduplication, nothing remembered"
            );
        }
        assert_eq!(stage.emitted(), 200_000);
    }
}