qua 0.44.0

Structure-aware query CLI for the Quarb engine
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288

//! --plan and --budget price the inputs from their catalogs before
//! anything is opened: a tarball of books by its headers, a
//! directory by its listing.

use std::path::{Path, PathBuf};

fn scratch() -> PathBuf {
    static N: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
    let n = N.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
    let dir = std::env::temp_dir().join(format!("qua-preflight-{}-{n}", std::process::id()));
    let _ = std::fs::remove_dir_all(&dir);
    std::fs::create_dir_all(dir.join("shelf")).unwrap();
    for (name, text) in [
        ("emma.txt", "Emma Woodhouse, handsome, clever, and rich"),
        ("iliad.txt", "Sing, O goddess, the anger of Achilles"),
        ("odyssey.txt", "Tell me, O Muse, of that ingenious hero"),
    ] {
        std::fs::write(dir.join("shelf").join(name), text).unwrap();
    }
    let gz = flate2::write::GzEncoder::new(
        std::fs::File::create(dir.join("books.tar.gz")).unwrap(),
        flate2::Compression::default(),
    );
    let mut ar = tar::Builder::new(gz);
    ar.append_dir_all("shelf", dir.join("shelf")).unwrap();
    ar.into_inner().unwrap().finish().unwrap();
    dir
}

fn qua(dir: &Path, args: &[&str]) -> (String, String, bool) {
    let out = std::process::Command::new(env!("CARGO_BIN_EXE_qua"))
        .env("QUARB_LOCALE", "C")
        .current_dir(dir)
        .args(args)
        .stdin(std::process::Stdio::null())
        .output()
        .unwrap();
    (
        String::from_utf8_lossy(&out.stdout).to_string(),
        String::from_utf8_lossy(&out.stderr).to_string(),
        out.status.success(),
    )
}

#[test]
fn a_shelf_is_priced_from_the_tarballs_headers() {
    let dir = scratch();
    let (out, err, ok) = qua(
        &dir,
        &[
            "--plan",
            "/books/*[::name = \"emma.txt\"]",
            "books=books.tar.gz",
        ],
    );
    assert!(ok, "{err}");
    // Three members and the folder entry: four entries.
    assert!(out.contains("nodes-min=0\n!int\nnodes-max=4"), "{out}");
    assert!(out.contains("confidence=bound"), "{out}");
    let (out, err, ok) = qua(&dir, &["--plan", "//*", "text:books.tar.gz"]);
    assert!(ok, "{err}");
    // Every member is a node of `//*`, with what the reading finds
    // beneath: a lower bound.
    assert!(
        out.contains(
            "nodes-min=4
"
        ),
        "{out}"
    );
    assert!(!out.contains("nodes-max="), "{out}");
    assert!(out.contains("latency=local"), "{out}");
    assert!(out.contains("confidence=bound"), "{out}");
}

#[test]
fn a_directory_is_priced_from_its_listing() {
    let dir = scratch();
    let (out, err, ok) = qua(&dir, &["--plan", "/*", "corpus:shelf"]);
    assert!(ok, "{err}");
    assert!(out.contains("nodes-max=3"), "{out}");
    assert!(out.contains("bytes-max=119"), "{out}");
}

#[test]
fn a_budget_refuses_before_opening_and_admits_within() {
    let dir = scratch();
    // The listing stops at the first file past the budget: the
    // refusal names what it reached, not the whole.
    let (_, err, ok) = qua(&dir, &["--budget", "bytes=50", "/*", "corpus:shelf"]);
    assert!(!ok);
    assert!(err.contains("reaches at least"), "{err}");
    assert!(err.contains("bytes; the budget allows 50"), "{err}");
    let (_, err, ok) = qua(&dir, &["--budget", "nodes=1", "/*", "corpus:shelf"]);
    assert!(!ok);
    assert!(
        err.contains("reaches at least 2 nodes; the budget allows 1"),
        "{err}"
    );
    let (out, err, ok) = qua(&dir, &["--budget", "bytes=500", "/*", "corpus:shelf"]);
    assert!(ok, "{err}");
    assert!(out.contains("emma.txt"), "{out}");
    // A plain file's node count is unknown before the reading: a
    // rows budget passes it unless strict.
    let (_, err, ok) = qua(&dir, &["--budget", "nodes=1", "/*", "shelf/emma.txt"]);
    assert!(ok, "{err}");
    let (_, err, ok) = qua(
        &dir,
        &["--budget", "nodes=1,strict", "/*", "shelf/emma.txt"],
    );
    assert!(!ok);
    assert!(err.contains("unbounded"), "{err}");
}

#[test]
fn a_budget_takes_units_and_two_inputs_are_priced_together() {
    let dir = scratch();
    let (_, err, ok) = qua(&dir, &["--budget", "bytes=1MB", "/*", "corpus:shelf"]);
    assert!(ok, "{err}");
    let (_, err, ok) = qua(&dir, &["--budget", "bytes=1x", "/*", "corpus:shelf"]);
    assert!(!ok);
    assert!(err.contains("--budget bytes"), "{err}");
    let (out, err, ok) = qua(
        &dir,
        &[
            "--plan",
            "//*",
            "text:shelf/emma.txt",
            "text:shelf/iliad.txt",
        ],
    );
    assert!(ok, "{err}");
    assert!(out.contains("(/plan/estimate)"), "{out}");
    assert!(out.contains("bytes-min=80"), "{out}");
}

/// A database file is its store's to price: the pre-flight leaves
/// it to the open, and the planned or budgeted run goes through the
/// engine (never the SQL pushdown, which would answer outright).
#[test]
fn a_database_is_priced_by_its_store_once_open() {
    let dir = scratch();
    let db = dir.join("loans.db");
    let conn = rusqlite::Connection::open(&db).unwrap();
    // Two indexes, as a large table always has: the statistics
    // then come one row per index, none with a null idx.
    conn.execute_batch(
        "CREATE TABLE loans (id INTEGER PRIMARY KEY, book TEXT, reader INTEGER);
         CREATE INDEX loans_book ON loans (book);
         CREATE INDEX loans_reader ON loans (reader);",
    )
    .unwrap();
    for i in 0..500 {
        conn.execute(
            "INSERT INTO loans (book) VALUES (?1)",
            [if i % 5 == 0 { "Emma" } else { "Iliad" }],
        )
        .unwrap();
    }
    // Before ANALYZE the store has no statistics: the plan carries
    // no estimate and a strict budget refuses as unknown.
    let (out, err, ok) = qua(&dir, &["--plan", "/loans/*", "loans.db"]);
    assert!(ok, "{err}");
    assert!(out.contains("[/plan/@steps]"), "{out}");
    assert!(!out.contains("(/plan/estimate)"), "{out}");
    let (_, err, ok) = qua(
        &dir,
        &[
            "--budget",
            "nodes=10,strict",
            "/loans/* @| count",
            "loans.db",
        ],
    );
    assert!(!ok);
    assert!(err.contains("unknown"), "{err}");
    conn.execute_batch("ANALYZE").unwrap();
    drop(conn);
    let (out, err, ok) = qua(&dir, &["--plan", "/loans/*", "loans.db"]);
    assert!(ok, "{err}");
    assert!(out.contains("nodes-max=500"), "{out}");
    let (_, err, ok) = qua(
        &dir,
        &["--budget", "nodes=10", "/loans/* @| count", "loans.db"],
    );
    assert!(!ok);
    assert!(
        err.contains("estimates 500 nodes; the budget allows 10"),
        "{err}"
    );
    let (out, err, ok) = qua(
        &dir,
        &["--budget", "nodes=1000", "/loans/* @| count", "loans.db"],
    );
    assert!(ok, "{err}");
    assert_eq!(out.trim(), "500");
}

#[test]
fn a_narrower_path_lowers_the_estimate() {
    let dir = scratch();
    std::fs::create_dir_all(dir.join("shelf/verse")).unwrap();
    std::fs::rename(
        dir.join("shelf/iliad.txt"),
        dir.join("shelf/verse/iliad.txt"),
    )
    .unwrap();
    // The whole tree, then one folder, then one file.
    let (out, err, ok) = qua(&dir, &["--plan", "//*", "corpus:shelf"]);
    assert!(ok, "{err}");
    assert!(out.contains("bytes-max=119"), "{out}");
    let (out, err, ok) = qua(&dir, &["--plan", "/verse//*", "corpus:shelf"]);
    assert!(ok, "{err}");
    assert!(
        out.contains("(/plan/estimate)\n!int\nnodes-min=1\n!int\nbytes-min=38\n!int\nbytes-max=38"),
        "{out}"
    );
    let (out, err, ok) = qua(
        &dir,
        &["--plan", "/verse/iliad.txt//paragraph", "corpus:shelf"],
    );
    assert!(ok, "{err}");
    assert!(
        out.contains("(/plan/estimate)\n!int\nnodes-min=0\n!int\nbytes-min=38\n!int\nbytes-max=38"),
        "{out}"
    );
    let (_, err, ok) = qua(
        &dir,
        &["--budget", "bytes=100", "/verse//*", "corpus:shelf"],
    );
    assert!(ok, "{err}");
    let (_, err, ok) = qua(&dir, &["--budget", "bytes=100", "//*", "corpus:shelf"]);
    assert!(!ok);
    assert!(err.contains("119 bytes"), "{err}");
}

#[test]
fn a_listing_prices_a_size_filter_and_statistics_price_a_key() {
    let dir = scratch();
    // A size is a quantity, so the filter says its unit (spec:
    // Relations: a bare number never compares with a dimensioned
    // value, and `::size > 40` prices to no node at all).
    let (out, err, ok) = qua(&dir, &["--plan", "//*<file>[::size > 40]", "corpus:shelf"]);
    assert!(ok, "{err}");
    assert!(out.contains("nodes-max=0\n"), "{out}");
    let (out, err, ok) = qua(
        &dir,
        &["--plan", "//*<file>[::size > 40_B]", "corpus:shelf"],
    );
    assert!(ok, "{err}");
    assert!(out.contains("trait=file"), "{out}");
    // Priced by the listing, still the engine's to run.
    assert!(out.contains("conjunct=::size > 40_B (unknown)\n"), "{out}");
    assert!(
        out.contains("nodes-min=0\n!int\nnodes-max=1\n!int\nbytes-min=42\n!int\nbytes-max=42"),
        "{out}"
    );
    // A store's statistics: one value of an indexed column selects
    // the index's rows per key.
    let db = dir.join("loans.db");
    let conn = rusqlite::Connection::open(&db).unwrap();
    conn.execute_batch(
        "CREATE TABLE loans (id INTEGER PRIMARY KEY, book TEXT, reader INTEGER);
         CREATE INDEX loans_book ON loans (book);
         CREATE INDEX loans_reader ON loans (reader);",
    )
    .unwrap();
    for i in 0..500 {
        conn.execute(
            "INSERT INTO loans (book, reader) VALUES (?1, ?2)",
            (if i % 5 == 0 { "Emma" } else { "Iliad" }, i % 50),
        )
        .unwrap();
    }
    conn.execute_batch("ANALYZE").unwrap();
    drop(conn);
    let (out, err, ok) = qua(&dir, &["--plan", "/loans/*[::reader = 7]", "loans.db"]);
    assert!(ok, "{err}");
    assert!(out.contains("conjunct=::reader = 7 (residual)\n"), "{out}");
    assert!(out.contains("nodes-min=0\n!int\nnodes-max=10\n"), "{out}");
    let (out, err, ok) = qua(&dir, &["--plan", "/loans/*[::book = \"Emma\"]", "loans.db"]);
    assert!(ok, "{err}");
    assert!(out.contains("(pushed)"), "{out}");
    assert!(out.contains("nodes-min=0\n!int\nnodes-max=250\n"), "{out}");
    let (out, err, ok) = qua(&dir, &["--plan", "/loans/7", "loans.db"]);
    assert!(ok, "{err}");
    assert!(out.contains("nodes-min=0\n!int\nnodes-max=1\n"), "{out}");
}