mariadb-cli 0.1.1

One-shot agent-native command line client for MariaDB servers
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
//! Proving that the single document is written as its rows arrive.
//!
//! # Why this file exists
//!
//! The single document is the built-in shape, so almost every caller reads it.
//! It was also the one shape a read could not be served as a flow in: the rows
//! were gathered so they could be serialised together. The common case was
//! therefore the expensive case, and the guarantee that a set larger than
//! memory can be read held for every shape except the one nobody had to ask
//! for.
//!
//! Writing the rows as they arrive is only worth anything if the answer is the
//! same answer, so the first proof below drives ONE read down BOTH paths and
//! compares the bytes against each other. It compares them against each other
//! rather than against a document written here, because a document written here
//! would fix an address, a version and a field order that belong to whichever
//! server this suite is pointed at, and it would go on passing after the two
//! paths had drifted apart — which is exactly the defect.
//!
//! The second proof measures the heap, because "streams" is a claim about cost
//! and nothing about the bytes can show it. The third reads the mark FR-132
//! asks for: a set cut part way through must never look whole. This shape
//! carries no field for that, and does not need one, because a cut flow leaves
//! a document that never closes.
//!
//! The counter the second proof reads is global to the process, so it also
//! counts what the other two allocate. That is why it asserts a bound rather
//! than an equality: the bound is far below what holding the rows would cost,
//! and far above anything two invocations of a subprocess can add.

// Each file here uses the part of the shared module that it needs, so a helper
// unused by this one is not dead code.
#[allow(dead_code)]
mod support;

use std::alloc::{GlobalAlloc, Layout, System};
use std::sync::atomic::{AtomicUsize, Ordering};
use support::server::{destination_or_skip, on_server};
use support::Sandbox;

/// Bytes handed out and not yet given back.
static LIVE: AtomicUsize = AtomicUsize::new(0);

/// The system allocator, counting what it lends.
struct Counting;

// SAFETY: every call is forwarded to the system allocator unchanged, and the
// only addition is an atomic counter, which allocates nothing itself.
unsafe impl GlobalAlloc for Counting {
    unsafe fn alloc(&self, layout: Layout) -> *mut u8 {
        let pointer = unsafe { System.alloc(layout) };
        if !pointer.is_null() {
            LIVE.fetch_add(layout.size(), Ordering::Relaxed);
        }
        pointer
    }

    unsafe fn dealloc(&self, pointer: *mut u8, layout: Layout) {
        LIVE.fetch_sub(layout.size(), Ordering::Relaxed);
        unsafe { System.dealloc(pointer, layout) };
    }

    unsafe fn realloc(&self, pointer: *mut u8, layout: Layout, new_size: usize) -> *mut u8 {
        let moved = unsafe { System.realloc(pointer, layout, new_size) };
        if !moved.is_null() {
            LIVE.fetch_sub(layout.size(), Ordering::Relaxed);
            LIVE.fetch_add(new_size, Ordering::Relaxed);
        }
        moved
    }
}

#[global_allocator]
static ALLOCATOR: Counting = Counting;

/// Bytes the process is holding right now.
fn live() -> usize {
    LIVE.load(Ordering::Relaxed)
}

/// A read of three rows already in the order any ordering would put them in.
///
/// Already ordered on purpose: the gathered path is reached by asking for an
/// order, and an order that moved a row would change the answer as well as the
/// path, leaving the comparison measuring two things at once.
const A_READ: &str = "SELECT 1 AS a UNION ALL SELECT 2 UNION ALL SELECT 3";

/// A read wide enough that a volume ceiling cuts it part way through.
const WIDER_THAN_ANY_CEILING: &str =
    "SELECT TABLE_NAME AS t, COLUMN_NAME AS c FROM information_schema.COLUMNS";

/// What asking for an order adds to the answer, and the only thing it adds.
///
/// An order over the whole set cannot be decided before the last row arrives,
/// so it is what the gathered path is reached by. It is also declared, which
/// FR-032 requires: a caller who asked for a flow and received a gathered
/// answer is told which operation cost it. That declaration is the difference
/// between the two documents, and it is discounted below because the request
/// itself put it there.
const DECLARED_BY_THE_ORDER: &str = r#","accumulated_for":["sort"]"#;

/// FR-032. One read answers with the same bytes whichever path carried it.
///
/// The gathered path is reached by asking for an order over the whole set,
/// which is a request the flow cannot serve. Everything else about the two
/// calls is identical, so everything else about the two answers has to be.
#[test]
fn one_read_answers_with_the_same_bytes_whichever_path_carried_it() {
    let Some(dsn) = destination_or_skip("one read answered by both paths") else {
        return;
    };
    let sandbox = Sandbox::new();

    let flowing = on_server(&sandbox, &dsn, &["--limit", "10", "query", "run", A_READ]);
    assert_eq!(flowing.code, 0, "the flow: {}", flowing.err);

    let gathered = on_server(
        &sandbox,
        &dsn,
        &["--limit", "10", "--sort", "a", "query", "run", A_READ],
    );
    assert_eq!(gathered.code, 0, "the gathered answer: {}", gathered.err);

    // The two really did travel by two different paths, which no comparison of
    // their bytes could show: agreeing is the whole point, so a comparison that
    // agreed because both had been gathered would pass having measured nothing.
    assert!(
        streams(&sandbox, &["--limit", "10", "query", "run", A_READ]),
        "the plain read must reach the flow, or this proof compares one path with itself"
    );
    assert!(
        !streams(
            &sandbox,
            &["--limit", "10", "--sort", "a", "query", "run", A_READ]
        ),
        "asking for an order must reach the gathered path"
    );

    assert!(
        gathered.out.contains(DECLARED_BY_THE_ORDER),
        "the order must declare itself, or the discount below is hiding something else: {}",
        gathered.out
    );
    let discounted = gathered.out.replace(DECLARED_BY_THE_ORDER, "");

    assert_eq!(
        flowing.out, discounted,
        "one read answered two ways: flowing `{}` against gathered `{}`",
        flowing.out, gathered.out
    );
    // A comparison of two empty channels also succeeds.
    assert!(
        flowing.out.contains(r#""rows":[{"a":1},{"a":2},{"a":3}]"#),
        "the rows themselves must be there: {}",
        flowing.out
    );
}

/// FR-032. The single document costs the same at the two hundred thousandth row
/// as at the first.
///
/// The bytes cannot show this, because a document assembled from gathered rows
/// and a document written as they arrive are the same document. What separates
/// them is what the process was holding while it wrote, so that is measured.
///
/// Two things are read, because either one alone would pass something wrong.
/// The first is that the flow settles and stays there, which a path that held
/// the rows could not. The second is that where it settled is far below what
/// the same rows gathered cost, without which a flow that held every row and
/// happened to allocate in one block would pass.
#[test]
fn the_single_document_costs_the_same_at_the_last_row_as_at_the_first() {
    use mariadb_cli::cli::globals::OutputFormat;
    use mariadb_cli::output::format::for_format;
    use mariadb_cli::output::stream::Stream;

    let rows = 200_000;
    let tail = mariadb_cli::output::Envelope::new("query run", serde_json::json!({}));
    let columns = vec!["id".to_owned(), "name".to_owned()];

    // The rows go to a channel that keeps nothing, because a buffer holding
    // them would measure the reader's cost rather than the flow's.
    let before = live();
    let mut flow = Stream::open(
        std::io::sink(),
        for_format(OutputFormat::Json),
        &tail,
        &columns,
    )
    .expect("the flow opens");

    let mut settled = 0;
    let mut peak = 0;
    for index in 0..rows {
        flow.write(&wire_row(index)).expect("the row is written");
        // The first few rows pay for the line buffer the writer reuses. From
        // there the flow must not grow, however many rows are still to come.
        // Written against the settling row rather than as a chain of ifs,
        // because the index stands in three places and only two of them were
        // spelled out. The rows before that point are the ones still allowed
        // to grow the flow, and leaving that case unwritten made it read like
        // an oversight rather than a decision. The oldest compiler this crate
        // supports asks for this shape; the newest one no longer does, and
        // this shape is what answers both.
        match index.cmp(&1_000) {
            std::cmp::Ordering::Less => {}
            std::cmp::Ordering::Equal => settled = live().saturating_sub(before),
            std::cmp::Ordering::Greater => peak = peak.max(live().saturating_sub(before)),
        }
    }
    assert_eq!(
        flow.finish(&tail, &[], None).expect("the flow closes"),
        rows,
        "every row reached the channel"
    );

    // The same rows gathered, which is what the single document used to do.
    let before_gathering = live();
    let held: Vec<mariadb_cli::output::Row> = (0..rows).map(wire_row).collect();
    let holding_them_all = live().saturating_sub(before_gathering);
    assert_eq!(held.len() as u64, rows);
    drop(held);

    // A bound rather than an equality: the counter is the whole process, and
    // the two proofs beside this one run subprocesses of their own. What they
    // add is measured in kilobytes; what holding the rows costs is measured in
    // tens of megabytes, so the two can never be confused for each other.
    let room = 4 * 1024 * 1024;
    assert!(
        peak <= settled + room,
        "the flow settled at {settled} bytes and peaked at {peak}, so it is holding rows \
         rather than writing them"
    );
    assert!(
        peak * 4 < holding_them_all,
        "the flow peaked at {peak} bytes and the same rows gathered hold {holding_them_all}; \
         a flow that costs what gathering costs is not a flow"
    );
}

/// FR-132. A document cut to a volume ceiling is still a document.
///
/// # Why this proof says the opposite of what it used to
///
/// It used to require the braces to stay open, on the reasoning that a document
/// has no equivalent of the closing record the line-oriented shapes carry, so
/// the mark of a cut had to be structural. That reasoning is coherent and it
/// lost to a plainer rule: this product's first promise about its primary
/// channel is that what leaves on it can be parsed. A ceiling asked for with
/// `--max-output-bytes` is not a failure — it is the caller saying how much it
/// wants — and answering a request with something no reader accepts punishes
/// the caller for asking.
///
/// The cut is still declared, in the two places a reader already looks: the
/// `applied` block carries `output_truncated` and how many were dropped, and
/// `gaps` names the object that lost rows. What changed is that noticing the
/// cut now needs a field to be read rather than a parse to fail.
///
/// The decision was the user's, taken on the contradiction between this proof
/// and the one in `the_cut_answer_still_parses.rs`, which held that a cut
/// answer must remain readable. Both could not stand.
#[test]
fn a_document_cut_part_way_through_is_still_a_document() {
    let Some(dsn) = destination_or_skip("a cut document that never closes") else {
        return;
    };
    let sandbox = Sandbox::new();

    // The volume ceiling stands in for the dropped channel: every way of
    // cutting a flow meets the same line, and this one cuts on demand rather
    // than whenever the network feels like it.
    let cut = on_server(
        &sandbox,
        &dsn,
        &[
            "--max-output-bytes",
            "2000",
            "query",
            "run",
            WIDER_THAN_ANY_CEILING,
        ],
    );
    assert_eq!(cut.code, 0, "a cut flow is still an answer: {}", cut.err);
    assert!(
        cut.out.starts_with(r#"{"command":"query run""#),
        "a cut flow still carries what it managed to write: {}",
        cut.out
    );
    let read: serde_json::Value = serde_json::from_str(cut.out.trim_end())
        .unwrap_or_else(|why| panic!("a cut answer must still be readable: {why}"));
    assert_eq!(
        read["applied"]["output_truncated"],
        serde_json::json!(true),
        "the cut is no longer visible in the structure, so it has to be \
         visible in the field, and it is not"
    );
    assert!(
        read["applied"]["dropped"].as_u64().is_some_and(|n| n > 0),
        "an answer that dropped rows must say how many it dropped"
    );

    // And the same read, uncut, is one document a reader accepts.
    let whole = on_server(&sandbox, &dsn, &["--limit", "5", "query", "run", A_READ]);
    assert_eq!(whole.code, 0, "the whole answer: {}", whole.err);
    let parsed = whole.answer();
    assert_eq!(parsed["data"]["rows"].as_array().expect("rows").len(), 3);
}

/// FR-032. A ceiling of bytes does not cost the flow.
///
/// # Why the proof about peak memory could not catch this
///
/// The proof above opens a `Stream` and writes two hundred thousand rows into
/// it, showing that the flow does not grow. It shows that about the flow, and
/// says nothing about which path an invocation is sent down — so a change that
/// sent a whole family of invocations to the gathered path instead would leave
/// it green.
///
/// That happened. A repair for a cut document routed `--max-output-bytes` to
/// the gathered path, and the measured cost was a peak of 88 MiB against 26 MiB
/// for the same two hundred thousand rows producing the same two thousand
/// bytes. Every one of the suite's proofs passed. The repair was replaced by
/// one that keeps the flow, and this is the proof that would have objected.
///
/// It asks the product's own gate rather than measuring bytes, for the reason
/// the sibling proof gives: the paths agreeing on their output is what makes
/// the output useless as evidence of which path ran.
#[test]
fn asking_for_a_ceiling_of_bytes_still_reaches_the_flow() {
    let sandbox = Sandbox::new();

    assert!(
        streams(
            &sandbox,
            &["--max-output-bytes", "2000", "query", "run", A_READ]
        ),
        "a ceiling of bytes sent this read to the path that holds every row \
         first, which costs the flow the one thing a flow is for"
    );

    // The line-oriented shape has always met a ceiling by flowing, and losing
    // that would be the same defect wearing a different shape.
    assert!(
        streams(
            &sandbox,
            &[
                "--output-format",
                "ndjson",
                "--max-output-bytes",
                "2000",
                "query",
                "run",
                A_READ
            ]
        ),
        "a ceiling of bytes sent the line-oriented read to the gathered path"
    );
}

/// Every condition the gate weighs, and which path it sends the call down.
///
/// The gate answers one question — flow or gather — out of several inputs, and
/// each of them was reached by a proof only if somebody happened to write one.
/// Two were. The rest were decided by code that nothing read back, which is the
/// arrangement that let a repair move a whole family of calls to the gathered
/// path with every proof still green.
///
/// Written as a table so that adding a condition to the gate and not adding a
/// row here is visible as an absence, rather than as nothing at all.
const AT_THE_GATE: &[(&str, &[&str], bool)] = &[
    ("a plain read", &["query", "run", A_READ], true),
    (
        "a ceiling of rows",
        &["--limit", "10", "query", "run", A_READ],
        true,
    ),
    (
        "a ceiling of bytes",
        &["--max-output-bytes", "2000", "query", "run", A_READ],
        true,
    ),
    (
        "a projection",
        &["--select", "a", "query", "run", A_READ],
        true,
    ),
    // An order is over the whole set, so the first row of the answer may be the
    // last one the server sends.
    ("an order", &["--sort", "a", "query", "run", A_READ], false),
    // A count and a summary are single values over a set that is not a set
    // until it is complete.
    ("a count", &["--count-only", "query", "run", A_READ], false),
    ("a summary", &["--summary", "query", "run", A_READ], false),
    // A signature covers the set as one object.
    (
        "a signature",
        &["--checksum", "query", "run", A_READ],
        false,
    ),
];

/// FR-032. The gate sends each kind of call down the path it belongs to.
///
/// Asked of the gate rather than measured from the output, because the two
/// paths agreeing on their bytes is the property being relied on and cannot
/// also be the evidence for which one ran.
#[test]
fn every_condition_the_gate_weighs_is_read_back() {
    let sandbox = Sandbox::new();

    let mut wrong = Vec::new();
    for (what, argv, expected) in AT_THE_GATE {
        let reached = streams(&sandbox, argv);
        if reached != *expected {
            let went = if reached { "flowed" } else { "gathered" };
            let meant = if *expected { "flow" } else { "gather" };
            wrong.push(format!("{what}: {went}, and it must {meant}"));
        }
    }

    assert!(
        wrong.is_empty(),
        "the gate sent these down the wrong path: {wrong:?}"
    );
}

/// One row, built the way the read path builds it.
fn wire_row(index: u64) -> mariadb_cli::output::Row {
    let mut row = mariadb_cli::output::Row::new();
    row.insert("id".to_owned(), serde_json::json!(index));
    row.insert("name".to_owned(), serde_json::json!(format!("row-{index}")));
    row
}

/// Whether this invocation reaches the flow rather than the gathered path.
///
/// Read from the product's own gate rather than guessed from the bytes,
/// because the two paths agreeing on the bytes is exactly what is being
/// proved and cannot also be the evidence for it.
fn streams(sandbox: &Sandbox, args: &[&str]) -> bool {
    use clap::Parser as _;
    let command_line: Vec<String> = ["mariadb-cli", "--config-dir"]
        .into_iter()
        .map(ToOwned::to_owned)
        .chain(std::iter::once(
            sandbox.path().to_string_lossy().into_owned(),
        ))
        .chain(args.iter().map(|piece| (*piece).to_owned()))
        .collect();
    let cli = mariadb_cli::cli::Cli::try_parse_from(command_line).expect("the invocation parses");
    let config =
        mariadb_cli::config::Config::load(Some(sandbox.path())).expect("the location loads");
    mariadb_cli::dispatch::flow::applies(&cli, &config)
}