macrame/metrics.rs
1//! What the write actor knows about its own latency (T1.4, D-079).
2//!
3//! # Why this exists
4//!
5//! [`crate::CHUNK_BUDGET`] is 3 ms and the crate has, until now, had exactly one
6//! way to find out whether that bound holds: run `benches/budgets.rs` on a
7//! synthetic fixture. That is a statement about a laptop, not about a database
8//! in use. D-059 already established that the bound does **not** hold on a large
9//! file, by a factor of 15, and it took a benchmark rewrite to notice — because
10//! nothing in the running system was counting.
11//!
12//! Tier 1's other three items are all "make the tail bounded". None of them can
13//! be validated in the field without something that measures the tail, which is
14//! why this is a precondition for them rather than a nice-to-have.
15//!
16//! # What is recorded, and what is deliberately not
17//!
18//! Four things, all of them per **actor turn** — one command, start to finish:
19//!
20//! - **queue depth** on both channels, sampled *before* the turn begins;
21//! - **hold duration**, bucketed, per command kind;
22//! - **holds over budget**, counted separately per kind;
23//! - **the longest hold since open**, with the kind that caused it.
24//!
25//! The hold is the whole turn, not the `execute` call's SQL. That is the
26//! quantity the budget is about: the SQLite write lock is not preemptible, so an
27//! interactive assertion arriving mid-turn waits for the turn, whatever the turn
28//! spent its time on.
29//!
30//! There is no per-command timestamp trail and no sampling of individual slow
31//! commands. That would be a tracing problem, and `tracing` is already a
32//! dependency — spans belong there. This module answers one question ("is the
33//! bound holding, and if not, which kind breaks it") in fixed memory, with no
34//! allocation on the actor's path.
35//!
36//! # The feature gate
37//!
38//! Behind `metrics`, which has been a **default** feature since 0.12.11
39//! (D-154): a crate whose contract is a latency bound must not ship a default
40//! build that cannot report whether the bound is met. `--no-default-features`
41//! still removes it. With the feature off, [`ActorMetrics`] is a
42//! zero-sized type whose methods compile away and [`HoldTimer::start`] does not
43//! read the clock — so the actor loop has **one** shape either way. That
44//! matters more than the nanoseconds: a `#[cfg]` in the loop body is how the
45//! instrumented and uninstrumented paths drift until only one of them is the one
46//! that runs.
47
48use std::time::Duration;
49
50/// The command kinds the actor can spend a turn on.
51///
52/// One flat enum across both channels rather than one per channel. The question
53/// this exists to answer is "which command broke the budget", and a reader
54/// looking at a 400 ms hold does not first want to know which queue it came off.
55/// Priority is a property of scheduling; kind is a property of cost.
56///
57/// # `#[non_exhaustive]`, added while it was still free (0.12.8, W4.2)
58///
59/// Adding a variant here is a **breaking change** without this attribute,
60/// because a downstream `match` on `CommandKind` would stop compiling. That is
61/// not hypothetical for this enum: [`crate::metrics::CommandKind::Rehydrate`]
62/// did not exist until 0.12.9 precisely because adding it was a break, and
63/// rehydration reported as `Archive` for several releases as a result. The
64/// codebase has already paid this cost once, which is the argument for paying
65/// the attribute now rather than deciding it at 1.0 when the cost is permanent.
66///
67/// Callers must therefore include a `_ =>` arm. In exchange, this enum can grow
68/// a variant for a command kind that does not exist yet without a major version.
69#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
70#[repr(u8)]
71#[non_exhaustive]
72pub enum CommandKind {
73 AssertEdge,
74 RetireEdge,
75 UpsertConcept,
76 WriteBulkAtomic,
77 RebuildCurrent,
78 RegisterModel,
79 Shutdown,
80 BulkImportChunk,
81 WriteConceptsChunk,
82 WriteAnalyticsChunk,
83 UpsertEmbeddingChunk,
84 Archive,
85 RebuildFts,
86 /// The **fill** half of a chunked shadow rebuild — `Begin` and every
87 /// `Fill` chunk (T1.2).
88 ///
89 /// Its own kind rather than folded into `RebuildCurrent`, because the two
90 /// have opposite latency profiles and the whole point of the chunked path
91 /// is that its turns are short — averaging them together would hide
92 /// exactly the improvement.
93 ///
94 /// **Fill-only since 0.14.16** ([D-233]). Through 0.14.15 this kind also
95 /// carried the swap turn, which is over budget by construction, so its
96 /// `over_budget` count was `N(rebuilds) + regressions` and could not be
97 /// decomposed — the counter was a constant, not a signal. The swap is
98 /// [`CommandKind::ShadowSwap`] now, and what is left here is the half that
99 /// is *meant* to fit [`crate::CHUNK_BUDGET`]. A nonzero count on this kind
100 /// is therefore a clean canary: a fill chunk ran long, which is a
101 /// regression and nothing else.
102 ///
103 /// [D-233]: ../docs/architecture/s13-decision-register.md#d-233
104 ShadowRebuild,
105 /// Refreshing the query planner's statistics (0.12.4, D-149).
106 ///
107 /// Its own kind rather than folded into `RebuildFts`, though both are
108 /// maintenance on derived state: this one is bounded by
109 /// `PRAGMA analysis_limit` and that one is bounded by the size of the
110 /// concept table, so averaging their holds together would describe neither.
111 Analyze,
112 /// Moving archived rows back into the hot file (0.12.9, W4.3, D-152).
113 ///
114 /// Its own kind at last. Through 0.12.8 this reported as
115 /// [`CommandKind::Archive`], on the stated ground that rehydration is the
116 /// archive path run backwards and shares its budget — true of the *budget*
117 /// and false of the *attribution*, which is what a metrics surface is for.
118 /// An operator reading a long `archive` hold could not tell whether the
119 /// database had archived anything at all, and the two move rows in opposite
120 /// directions.
121 ///
122 /// The real reason it stayed folded was that adding a variant was a
123 /// breaking change. `#[non_exhaustive]` (W4.2) is what removed that
124 /// obstacle, and this variant is the first thing it bought — which is also
125 /// the evidence that the attribute was worth adding rather than a
126 /// precaution against a hypothetical.
127 ///
128 /// **Appended at the end**, per [`CommandKind::index`]: the position of
129 /// every existing variant is a persisted contract in two languages.
130 Rehydrate,
131 /// An explicit `PRAGMA wal_checkpoint` (0.12.13, W5.2, D-156).
132 ///
133 /// Its own kind because it is the one actor turn that is **not** a
134 /// transaction: it moves frames from the WAL back into the main database
135 /// file, and its duration is a function of how much WAL has accumulated
136 /// rather than of anything the caller passed. Folding it into any existing
137 /// kind would make that kind's hold distribution bimodal for a reason no
138 /// dashboard could recover.
139 ///
140 /// **Appended at the end**, per [`CommandKind::index`].
141 Checkpoint,
142 /// `PRAGMA optimize` — re-analysing only what SQLite believes has drifted
143 /// (0.13.24, W10.5, D-197).
144 ///
145 /// Split out of [`CommandKind::Analyze`], which covered both from 0.12.4 to
146 /// 0.13.23. The split is [`CommandKind::Rehydrate`]'s lesson applied before
147 /// the fact rather than after it: [D-168] refused to decide `Analyze`'s
148 /// budget exemption *because* the kind was shared, since a judgement made
149 /// about the explicit call would have landed on the automatic one —
150 /// `close()` runs `optimize()` unconditionally — without ever being made
151 /// about it.
152 ///
153 /// The two also have genuinely different hold distributions, which is the
154 /// same argument [`CommandKind::ShadowRebuild`] is separate on.
155 /// [`crate::Database::analyze`] does the work unconditionally and its hold
156 /// tracks the table. This one is a no-op when nothing has moved, so its
157 /// distribution is bimodal by design and averaging the two together
158 /// describes neither.
159 ///
160 /// **Appended at the end**, per [`CommandKind::index`].
161 ///
162 /// [D-168]: ../docs/architecture/s13-decision-register.md#d-168
163 Optimize,
164 /// Registering a lineage (0.14.7, §15.4).
165 ///
166 /// Its own kind rather than folded into `AssertEdge`, though both are one
167 /// small transaction: a fork writes to `branches` and nothing else, so its
168 /// hold is the floor an actor turn can have, and averaging it into a
169 /// command that touches four tables would flatter that command's numbers.
170 ///
171 /// Last in declaration order because that order is a persisted contract and
172 /// **new variants go at the end** — see [`CommandKind::index`]. Grouping it
173 /// next to `RegisterModel`, which is where it belongs by kind, would have
174 /// renumbered nine counters and relabelled the Python histogram's axes.
175 Fork,
176 /// Forgetting a lineage (0.14.13, §15.4, D-230).
177 ///
178 /// Its own kind rather than folded into [`CommandKind::Archive`], on
179 /// [D-152]'s finding rather than on a fresh argument: the budget really is
180 /// shared and the attribution is not, and an operator reading a long
181 /// `archive` hold could not tell whether the database had archived a
182 /// backlog of closed intervals or dropped an abandoned branch. The two also
183 /// have unrelated cost curves — one is a function of how long it has been
184 /// since the last run, the other of how much was written on one branch.
185 ///
186 /// At the end of the declaration order, per [`CommandKind::index`].
187 ///
188 /// [D-152]: ../docs/architecture/s13-decision-register.md#d-152
189 ArchiveBranch,
190 /// The **swap** turn of a chunked shadow rebuild (0.14.16, D-233).
191 ///
192 /// Split out of [`CommandKind::ShadowRebuild`], which covered both halves
193 /// from 0.6.0 to 0.14.15. This is the third instance of one shape —
194 /// [`CommandKind::Rehydrate`] out of `Archive` ([D-152]),
195 /// [`CommandKind::Optimize`] out of `Analyze` ([D-197]), this — so the
196 /// class is named where it can be seen: **one `CommandKind`, one
197 /// structural hold distribution.** A kind covering two is a defect on
198 /// arrival, to be split in review rather than found by probe.
199 ///
200 /// Here the bimodality is structural rather than workload-dependent, which
201 /// is what makes it the clearest instance of the three. Index names are
202 /// global and SQLite has no `ALTER INDEX … RENAME`, so the shadow cannot
203 /// carry `idx_lc_traversal_cover` while the live table still holds that
204 /// name — the swap is where all three indexes get built, under the write
205 /// lock. [D-082](../docs/architecture/s13-decision-register.md#d-082)
206 /// measured it at **46.8 ms**, 15.6× the budget, and it grows with the
207 /// table.
208 ///
209 /// **Exempt**, unlike its fill half — see
210 /// [`CommandKind::exempt_from_budget`], where the criterion is stated.
211 ///
212 /// At the end of the declaration order, per [`CommandKind::index`].
213 ///
214 /// [D-197]: ../docs/architecture/s13-decision-register.md#d-197
215 ShadowSwap,
216}
217
218impl CommandKind {
219 /// Every kind, in declaration order. Indexing into the per-kind arrays is by
220 /// position in this slice, so the two must not drift — which is why the
221 /// arrays are sized from `ALL.len()` rather than from a hand-written count.
222 pub const ALL: &'static [CommandKind] = &[
223 CommandKind::AssertEdge,
224 CommandKind::RetireEdge,
225 CommandKind::UpsertConcept,
226 CommandKind::WriteBulkAtomic,
227 CommandKind::RebuildCurrent,
228 CommandKind::RegisterModel,
229 CommandKind::Shutdown,
230 CommandKind::BulkImportChunk,
231 CommandKind::WriteConceptsChunk,
232 CommandKind::WriteAnalyticsChunk,
233 CommandKind::UpsertEmbeddingChunk,
234 CommandKind::Archive,
235 CommandKind::RebuildFts,
236 CommandKind::ShadowRebuild,
237 CommandKind::Analyze,
238 CommandKind::Rehydrate,
239 CommandKind::Checkpoint,
240 CommandKind::Optimize,
241 CommandKind::Fork,
242 CommandKind::ArchiveBranch,
243 CommandKind::ShadowSwap,
244 ];
245
246 pub const COUNT: usize = CommandKind::ALL.len();
247
248 /// This kind's slot in the per-kind arrays.
249 ///
250 /// # Declaration order is a persisted contract (0.12.8, W4.2)
251 ///
252 /// `self as usize` means the **order of the variants above** is the order of
253 /// every per-kind array in this module, and the compiler cannot catch a
254 /// change to it. Reordering the enum silently reassigns every counter to a
255 /// different command: the code compiles, the tests pass, and a histogram
256 /// read after the change attributes `archive`'s holds to `rebuild_fts`.
257 ///
258 /// **New variants go at the end**, always — including at the end of
259 /// [`CommandKind::ALL`], whose order is what `as_str()` and the Python
260 /// surface enumerate. This binds Python too: `BUCKET_BOUNDS_MICROS` is a
261 /// module constant there and `KindMetrics` is built by position, so a
262 /// reorder here relabels axes in a language the Rust compiler is not
263 /// looking at.
264 ///
265 /// `#[repr(u8)]` is on the enum for the same reason — it pins the
266 /// discriminants to the declaration order rather than leaving them to the
267 /// compiler — but it pins them to whatever the order *is*, so it does not
268 /// make a reorder safe. Only this rule does.
269 pub const fn index(self) -> usize {
270 self as usize
271 }
272
273 pub const fn as_str(self) -> &'static str {
274 match self {
275 CommandKind::AssertEdge => "assert_edge",
276 CommandKind::RetireEdge => "retire_edge",
277 CommandKind::UpsertConcept => "upsert_concept",
278 CommandKind::WriteBulkAtomic => "write_bulk_atomic",
279 CommandKind::RebuildCurrent => "rebuild_current",
280 CommandKind::RegisterModel => "register_model",
281 CommandKind::Shutdown => "shutdown",
282 CommandKind::BulkImportChunk => "bulk_import_chunk",
283 CommandKind::WriteConceptsChunk => "write_concepts_chunk",
284 CommandKind::WriteAnalyticsChunk => "write_analytics_chunk",
285 CommandKind::UpsertEmbeddingChunk => "upsert_embedding_chunk",
286 CommandKind::Archive => "archive",
287 CommandKind::RebuildFts => "rebuild_fts",
288 CommandKind::ShadowRebuild => "shadow_rebuild",
289 CommandKind::Analyze => "analyze",
290 CommandKind::Rehydrate => "rehydrate",
291 CommandKind::Checkpoint => "checkpoint",
292 CommandKind::Optimize => "optimize",
293 CommandKind::Fork => "fork",
294 CommandKind::ArchiveBranch => "archive_branch",
295 CommandKind::ShadowSwap => "shadow_swap",
296 }
297 }
298
299 /// Whether this kind is exempt from [`crate::CHUNK_BUDGET`] by contract.
300 ///
301 /// The exemptions are the table in `CHUNK_BUDGET`'s own rustdoc, and they
302 /// are carried here so a dashboard can separate "the budget is being
303 /// broken" from "the budget does not apply and never claimed to". Counting
304 /// an `archive` as a budget violation would make the violation count useless
305 /// on any database that archives.
306 ///
307 /// The two lists must agree, and since 0.12.9 they are tied together in
308 /// both directions by `the_budget_exemptions_and_their_documented_table_agree`
309 /// — the extra-row direction being the one worth having, since a table row
310 /// with no code behind it promises a caller an exemption the violation
311 /// counter is about to disagree with.
312 ///
313 /// # The criterion, stated at last (0.14.16, W12.16, [D-233])
314 ///
315 /// The register applied one rule three times without naming it, and naming
316 /// it is what let the fourth case be decided rather than argued.
317 ///
318 /// > **Exempt means the chunk bound does not apply: the operation is atomic
319 /// > by necessity and has no smaller unit. Counted means the bound applies,
320 /// > so exceeding it is information.**
321 ///
322 /// Every exemption on this list was argued that way in its own release,
323 /// whatever the summary sentence said afterwards.
324 /// [`CommandKind::WriteBulkAtomic`] ([D-014]) is one statement, and is the
325 /// kind that exists precisely because the chunked variant is *not* atomic.
326 /// [`CommandKind::Archive`] ([D-012]) and [`CommandKind::ArchiveBranch`]
327 /// delete a consistent set or none of it.
328 /// [`CommandKind::RebuildCurrent`] ([D-023]) re-derives a whole projection
329 /// in one transaction. [`CommandKind::Rehydrate`] ([D-152]) is one
330 /// unchunked transaction moving rows back across the file boundary.
331 /// [`CommandKind::Checkpoint`] ([D-156]) is a WAL boundary. None of them
332 /// has a smaller unit to chunk *into*, so 3 ms is not a bound they failed —
333 /// it is a bound that was never about them.
334 /// [`CommandKind::Analyze`] and [`CommandKind::Optimize`] ([D-197]) do have
335 /// one: they are bounded work that can take longer or shorter, so exceeding
336 /// is a fact about this database and worth counting.
337 ///
338 /// # The criterion took three tries, and the two that failed are the useful part
339 ///
340 /// **v1 — *expected-on-healthy is exempt, workload-dependent is not*.**
341 /// Falsified by reading [D-197] closely rather than by any new measurement:
342 /// `Optimize` **runs on every close** and stays counted. If expectedness
343 /// decided the question, `Optimize` would be exempt and it is not.
344 ///
345 /// **v2 — *if `over_budget` can differ from `turns`, count it*.** Falsified
346 /// by three of the exemptions themselves: an `Archive` with nothing
347 /// archivable, a `Rehydrate` of a single row and a `Checkpoint` on an empty
348 /// WAL all come in *under* budget, so their counters can differ from their
349 /// turn counts and the rule would un-exempt all three.
350 ///
351 /// Both were **observational** — read off the outcomes the existing
352 /// exemptions happened to produce, and so decidable only after the fact.
353 /// Inapplicability is decidable at design time from what the operation *is*,
354 /// which is what a criterion has to be if it is to settle the next case
355 /// rather than rationalise the last one.
356 ///
357 /// Expected-on-healthy survives as **corroboration, not definition**: a kind
358 /// with no smaller unit usually does exceed on every healthy database, so
359 /// the symptom is a fair sanity check on the diagnosis. `Optimize` is
360 /// exactly the case that shows why it cannot be the test itself.
361 ///
362 /// `over_budget` is incremented once per turn that exceeds, so it counts
363 /// **occurrences and not magnitude**. That is the fact the criterion turns
364 /// on: a kind whose every turn exceeds contributes a constant to
365 /// [`MetricsSnapshot::budget_violations`] and moves not at all when the
366 /// hold doubles. Growth is visible in this kind's histogram and
367 /// [`KindSnapshot::longest`], which no exemption touches.
368 ///
369 /// # The two halves of a shadow rebuild land on opposite sides of it
370 ///
371 /// [`CommandKind::ShadowSwap`] is exempt: ≥ 15.6× by construction ([D-082]
372 /// measured 46.8 ms against a 3 ms budget), with no healthy state in which
373 /// it fits, and routine — the crate's own end-to-end suite triggers one.
374 /// Counting it would put a permanent `N(rebuilds)` in the violation list
375 /// of every database that has ever repaired its projection, which is
376 /// [`CommandKind::Rehydrate`]'s argument exactly.
377 ///
378 /// [`CommandKind::ShadowRebuild`] — the fill half — is **not** exempt, and
379 /// that is the half [D-082] was protecting when it refused to exempt the
380 /// merged kind: *"exempting the kind would hide the first fact to excuse
381 /// the second."* The goal is reaffirmed and the mechanism superseded. The
382 /// split protects fill structurally, where non-exemption of the merged
383 /// kind only protected it in principle: a fill regression used to arrive
384 /// as `+1` on a counter that already read `N(rebuilds)`, and now it is the
385 /// only thing that can move `shadow_rebuild` off zero at all.
386 ///
387 /// `a_swap_over_budget_is_not_a_violation` is what keeps this honest, and
388 /// its fixture is the load-bearing part: it seeds enough of a graph to put
389 /// the swap **over** the budget, asserts that first, and only then asserts
390 /// the swap's own count is zero. The obvious form — run a rebuild, assert
391 /// the violation list is empty — is worthless twice over. On a small
392 /// fixture the swap finishes inside 3 ms and the assertion passes whether
393 /// the kind is exempt or not; on a real one the *fill* chunks exceed the
394 /// budget legitimately (3.14 ms at 200 keys in a debug build), so an empty
395 /// list is a property of small fixtures rather than of rebuilds.
396 ///
397 /// The two tests are **one instrument with two asymmetric halves**, and it
398 /// is worth being exact about which owns what.
399 /// `a_swap_over_budget_is_not_a_violation` owns **narrowing**: re-count the
400 /// swap and its assertion moves off zero. It cannot own widening, because
401 /// widening an exemption only ever *removes* entries from
402 /// [`MetricsSnapshot::budget_violations`] — an assertion that a count is
403 /// zero stays green under every widening, including one that swallows the
404 /// fill half whole. **Widening is owned by
405 /// `a_long_fill_is_a_violation_and_a_long_swap_is_not` below and by nothing
406 /// else**, because forging a long fill and asserting it **is** counted is
407 /// the only shape of assertion a widening can break.
408 ///
409 /// # Any claim about fill and this budget must name a build mode and a fixture size
410 ///
411 /// At fixture scale the 3 ms bound sits **inside** fill variance rather
412 /// than above it, so the same assertion is true or false depending on how
413 /// the binary was compiled and how much graph it was handed. Debug
414 /// especially: 200 keys × 4 generations puts the longest fill at 3.14 ms —
415 /// one violation, the counter working — while a release build of the same
416 /// shape stays under. A test that asserts anything about fill overages is
417 /// therefore asserting something about *its own fixture and profile*, and
418 /// has to say which. The swap is the opposite and that is why the exemption
419 /// is testable at all: it exceeds by 15.6× and it exceeds in every mode.
420 ///
421 /// [D-012]: ../docs/architecture/s13-decision-register.md#d-012
422 /// [D-014]: ../docs/architecture/s13-decision-register.md#d-014
423 /// [D-023]: ../docs/architecture/s13-decision-register.md#d-023
424 /// [D-082]: ../docs/architecture/s13-decision-register.md#d-082
425 /// [D-152]: ../docs/architecture/s13-decision-register.md#d-152
426 /// [D-156]: ../docs/architecture/s13-decision-register.md#d-156
427 /// [D-197]: ../docs/architecture/s13-decision-register.md#d-197
428 /// [D-233]: ../docs/architecture/s13-decision-register.md#d-233
429 ///
430 /// # `Rehydrate` is exempt, and splitting it out is what made that a
431 /// decision rather than an accident (0.12.9, W4.3, D-152)
432 ///
433 /// Until 0.12.8 rehydration reported as [`CommandKind::Archive`] and was
434 /// therefore exempt **by inheritance** — nobody had decided it, it fell out
435 /// of the borrowed kind. Giving it its own variant would have silently
436 /// flipped it to non-exempt, and since a rehydrate is one unchunked
437 /// transaction moving rows back across the file boundary, every single one
438 /// would have counted as a budget violation. The violation count would then
439 /// have become useless on any database that rehydrates, which is precisely
440 /// the failure the `Archive` exemption exists to prevent, arriving by the
441 /// back door of a change made for attribution.
442 ///
443 /// So it is exempt, on the merits and now on the record: rehydration is the
444 /// archive path run backwards and makes the same claim about its hold —
445 /// that it is bulk movement with no latency bound, and that the caller asked
446 /// for it explicitly.
447 ///
448 /// # Neither [`CommandKind::Analyze`] nor [`CommandKind::Optimize`] is
449 /// exempt, and since 0.13.24 those are two decisions (W10.5, D-197)
450 ///
451 /// They were one kind from 0.12.4 to 0.13.23, and [D-168] declined to decide
452 /// the exemption *because* they were: `Analyze` covered
453 /// [`crate::Database::optimize`] too, `close()` calls that unconditionally,
454 /// and so a judgement made about the explicit call would have landed on the
455 /// automatic one without ever being made about it. That is
456 /// [`CommandKind::Rehydrate`]'s lesson above arriving from the other
457 /// direction — there a shared kind *granted* an exemption nobody had
458 /// decided; here one would have *laundered* one. W10.5 split the kind so
459 /// each could be answered on its own evidence. Both answers came back the
460 /// same and the reasons are different, which is the whole reason the split
461 /// had to come first.
462 ///
463 /// **[`CommandKind::Analyze`] cannot state a `Bound`, so it cannot have a
464 /// row.** `ANALYZE` is one indivisible statement whose cost is set by data
465 /// volume — measured at **5.26 ms at 10,000 edges and 19.1 ms at 40,000**
466 /// against a 3 ms budget (`examples/analyze_hold.rs`, [D-166]). Every call
467 /// is a violation and always will be. `Checkpoint`'s bound is frames
468 /// accumulated since the last one; `Archive`'s is the session's row count.
469 /// The honest entry here would be "the size of the table, damped 3–4× by
470 /// `analysis_limit`", which is not a bound but the absence of one, and a
471 /// row that cannot fill that column is this table admitting the thing it
472 /// exists to prevent.
473 ///
474 /// **[`CommandKind::Optimize`] is not exempt for the opposite reason: its
475 /// violations are rare and they are the informative ones.** Measured
476 /// (`examples/optimize_hold.rs`, 40,000 edges): **10.7 ms the first time on
477 /// a database that has never been analysed, and 90–220 µs every time
478 /// after** — comfortably inside the budget, including immediately after a
479 /// bulk load that doubled the ledger. It is over budget only when it
480 /// actually re-analyses something, and then it is over by a lot: **460 ms**
481 /// once the table had grown 25× and SQLite's staleness ratio finally
482 /// fired. So the count is bimodal by construction and it is *reporting*
483 /// rather than complaining: an `optimize` in `budget_violations()` marks
484 /// the calls that did work, which is exactly what an operator wants to
485 /// know and exactly what exempting the kind would delete.
486 ///
487 /// **The violations are expected and must not be "fixed" by lowering
488 /// [`crate::schema::ddl::ANALYSIS_LIMIT`].** That would buy the number by
489 /// sampling too little to separate the two `source_id`-leading indices,
490 /// which is the entire purpose of having statistics ([D-149]).
491 ///
492 /// [D-149]: ../docs/architecture/s13-decision-register.md#d-149
493 /// [D-166]: ../docs/architecture/s13-decision-register.md#d-166
494 /// [D-168]: ../docs/architecture/s13-decision-register.md#d-168
495 pub const fn exempt_from_budget(self) -> bool {
496 matches!(
497 self,
498 CommandKind::WriteBulkAtomic
499 | CommandKind::Archive
500 | CommandKind::RebuildCurrent
501 | CommandKind::Rehydrate
502 | CommandKind::ArchiveBranch
503 | CommandKind::Checkpoint
504 | CommandKind::ShadowSwap
505 )
506 }
507}
508
509impl std::fmt::Display for CommandKind {
510 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
511 f.write_str(self.as_str())
512 }
513}
514
515/// Upper bounds of the hold-duration histogram, in microseconds.
516///
517/// `3_000` is [`crate::CHUNK_BUDGET`] exactly, so the bucket boundary and the
518/// bound are the same number and a reader does not have to interpolate to answer
519/// "what fraction of turns fit". The tail runs to 1 s because D-059's measured
520/// worst case was 45 ms and `rebuild_current` at 40K rows is 318 ms (D-077) —
521/// a range this has to cover without saturating.
522///
523/// Anything above the last bound lands in the overflow bucket, which is why
524/// [`KindSnapshot::buckets`] is one longer than this slice.
525pub const BUCKET_BOUNDS_MICROS: &[u64] = &[
526 100, 300, 1_000, 3_000, 10_000, 30_000, 100_000, 300_000, 1_000_000,
527];
528
529/// Number of histogram buckets, including the overflow bucket.
530pub const BUCKET_COUNT: usize = BUCKET_BOUNDS_MICROS.len() + 1;
531
532#[allow(dead_code)] // used by `imp` under `metrics`, and by the tests always
533fn bucket_of(micros: u64) -> usize {
534 // Linear scan over nine bounds. A binary search here would be slower in
535 // practice and this runs once per actor turn, against a turn measured in
536 // microseconds at best.
537 BUCKET_BOUNDS_MICROS
538 .iter()
539 .position(|&bound| micros <= bound)
540 .unwrap_or(BUCKET_BOUNDS_MICROS.len())
541}
542
543/// Times one actor turn.
544///
545/// # This clock is no longer optional (0.12.0, W1)
546///
547/// Until 0.11.0 the field was `#[cfg(feature = "metrics")]` and `elapsed()`
548/// returned `Duration::ZERO` in a default build: the reading existed only to
549/// feed [`ActorMetrics::record_hold`]'s histogram, so a build that did not keep
550/// the histogram had no reason to read a clock.
551///
552/// The chunk loop changed what the reading is *for*. A chunk's measured hold is
553/// now the input to the next chunk's size (`connection::next_chunk_size`, named
554/// in prose because it is private — D-144), which means it is a control signal
555/// in every build and not an observation in some of them. Left gated, `bulk_import` would have sized its chunks off
556/// `Duration::ZERO` — a value that reads as "comfortably under budget" — and
557/// grown every chunk to the ceiling, in exactly the builds nobody was measuring.
558///
559/// So the clock is unconditional and **only the histogram is still gated**:
560/// `record_hold` remains a no-op without the feature. What that costs is one
561/// `Instant::now()` pair per actor turn — tens of nanoseconds against a turn
562/// measured in microseconds at best, and the same reasoning §5.1.5 uses to
563/// decide that a channel hop is free beside a chunk.
564///
565/// It stays a type rather than a bare `Instant::now()` in the loop because the
566/// ordering guarantee in [`crate::connection`]'s `Turn` is attached to it.
567pub struct HoldTimer {
568 start: std::time::Instant,
569}
570
571impl HoldTimer {
572 #[inline]
573 pub fn start() -> Self {
574 Self {
575 start: std::time::Instant::now(),
576 }
577 }
578
579 #[inline]
580 pub fn elapsed(&self) -> Duration {
581 self.start.elapsed()
582 }
583}
584
585// ---------------------------------------------------------------------------
586// Instrumented implementation
587// ---------------------------------------------------------------------------
588
589#[cfg(feature = "metrics")]
590mod imp {
591 use super::{bucket_of, CommandKind, BUCKET_COUNT};
592 use std::sync::atomic::{AtomicU64, Ordering};
593 use std::time::Duration;
594
595 /// One kind's counters. All `Relaxed`: these are statistics, and ordering
596 /// them against each other would buy a consistency no reader needs and cost
597 /// fences on the write path.
598 #[derive(Debug, Default)]
599 struct Kind {
600 turns: AtomicU64,
601 total_micros: AtomicU64,
602 over_budget: AtomicU64,
603 /// This kind's own high-water mark, in µs.
604 ///
605 /// Not redundant with the global `longest`. That one names a single
606 /// command, so on any real database it names whichever kind is slowest
607 /// overall — and the question "did windowing shrink the archive's worst
608 /// hold" cannot be answered by a counter that a bulk import wins. No
609 /// packing needed here: the kind is the array index.
610 longest_micros: AtomicU64,
611 buckets: [AtomicU64; BUCKET_COUNT],
612 }
613
614 /// Live counters, shared between the actor and the handle.
615 ///
616 /// Fixed size, no allocation, no lock. The actor updates; anyone may read.
617 #[derive(Debug, Default)]
618 pub struct ActorMetrics {
619 kinds: [Kind; CommandKind::COUNT],
620 /// Packed `micros << 8 | kind`, so the longest hold and the kind that
621 /// caused it are read and written as **one** value. Two atomics would
622 /// let a reader see a duration from one turn beside a kind from
623 /// another — a rare wrong answer to exactly the question this field
624 /// exists to answer. 2^56 µs is over two thousand years.
625 ///
626 /// **The duration must occupy the high bits.** The update is a
627 /// `fetch_max` on the packed word, so whichever field is packed high is
628 /// the one being compared. The first version of this had the kind up
629 /// there, which made the "longest hold" the hold with the largest
630 /// *enum index* — a 3 ms `write_concepts_chunk` beat a 10 ms
631 /// `rebuild_current` because its variant is declared later. It was
632 /// `actor_metrics_tests` that caught it, not the unit tests, because
633 /// nothing in the arithmetic is wrong: the packing is only incorrect in
634 /// the presence of the atomic operation it exists to serve.
635 longest: AtomicU64,
636 /// Loop iterations, which is **not** the number of turns taken.
637 ///
638 /// The depth sample happens at the top of the loop, before `select!`
639 /// blocks — so an idle actor has already counted the iteration for a
640 /// command that has not arrived. That is right for depth (the sample is
641 /// "what was queued when I went looking") and wrong for turns, which is
642 /// why [`MetricsSnapshot::turns`] is the sum of the per-kind counters
643 /// instead. Conflating the two made `turns` permanently one too high and
644 /// disagree with its own breakdown.
645 depth_samples: AtomicU64,
646 high_depth_sum: AtomicU64,
647 high_depth_max: AtomicU64,
648 low_depth_sum: AtomicU64,
649 low_depth_max: AtomicU64,
650 /// Turns where the actor took high-priority work while low-priority
651 /// work was already queued (0.12.10, W4.4, D-153).
652 ///
653 /// The `biased` `select!` in `run_writer_actor` has **no floor**:
654 /// sustained high-priority traffic can hold the low tier off
655 /// indefinitely, and nothing has ever said whether that happens. This
656 /// is the numerator of that question — how often the choice went
657 /// against the low tier at all.
658 low_starved_turns: AtomicU64,
659 /// The current unbroken run of such turns. Reset to zero the moment
660 /// low-priority work is taken.
661 ///
662 /// Not exposed; it is the state [`Self::low_starved_run_max`] is a
663 /// high-water mark of. A live value would be read at an arbitrary point
664 /// in a run and mean nothing.
665 low_starved_run: AtomicU64,
666 /// The longest such run since open, which is the number that answers the
667 /// question.
668 ///
669 /// A large `low_starved_turns` on a busy database is unremarkable — it
670 /// says the high tier is being used, which is what the tier is for. A
671 /// large *run* says one specific low-priority command waited that many
672 /// turns, and it is the only one of the two that can distinguish
673 /// "prioritised" from "starved".
674 low_starved_run_max: AtomicU64,
675 }
676
677 const MICROS_SHIFT: u32 = 8;
678 const KIND_MASK: u64 = (1 << MICROS_SHIFT) - 1;
679
680 impl ActorMetrics {
681 pub fn new() -> Self {
682 Self::default()
683 }
684
685 /// Sample both queue depths. Called before the turn, not after: after
686 /// the turn the queue reflects what arrived *during* it, which is a
687 /// different and much less useful quantity.
688 #[inline]
689 pub fn record_turn(&self, high_depth: usize, low_depth: usize) {
690 self.depth_samples.fetch_add(1, Ordering::Relaxed);
691 for (sum, max, depth) in [
692 (
693 &self.high_depth_sum,
694 &self.high_depth_max,
695 high_depth as u64,
696 ),
697 (&self.low_depth_sum, &self.low_depth_max, low_depth as u64),
698 ] {
699 sum.fetch_add(depth, Ordering::Relaxed);
700 max.fetch_max(depth, Ordering::Relaxed);
701 }
702 }
703
704 /// Record which tier the `select!` chose, and what was waiting.
705 ///
706 /// `low_queued` is the depth sampled *before* the `select!`, so it is
707 /// the backlog the turn found on arrival. By the time a high-priority
708 /// arm fires the low queue may have grown; using the pre-select reading
709 /// keeps this consistent with every other depth figure in this module
710 /// and makes the counter conservative — it never invents starvation
711 /// from work that arrived after the choice was made.
712 ///
713 /// A low-priority turn resets the run rather than decrementing it: the
714 /// question is "how many turns did one low-priority command wait", and
715 /// that is a run length, not a balance.
716 #[inline]
717 pub fn record_priority_choice(&self, took_high: bool, low_queued: usize) {
718 if took_high && low_queued > 0 {
719 self.low_starved_turns.fetch_add(1, Ordering::Relaxed);
720 let run = self.low_starved_run.fetch_add(1, Ordering::Relaxed) + 1;
721 self.low_starved_run_max.fetch_max(run, Ordering::Relaxed);
722 } else if !took_high {
723 self.low_starved_run.store(0, Ordering::Relaxed);
724 }
725 }
726
727 #[inline]
728 pub fn record_hold(&self, kind: CommandKind, held: Duration) {
729 let micros = held.as_micros().min(super::MICROS_CEILING as u128) as u64;
730 let k = &self.kinds[kind.index()];
731 k.turns.fetch_add(1, Ordering::Relaxed);
732 k.total_micros.fetch_add(micros, Ordering::Relaxed);
733 k.buckets[bucket_of(micros)].fetch_add(1, Ordering::Relaxed);
734 k.longest_micros.fetch_max(micros, Ordering::Relaxed);
735 if !kind.exempt_from_budget() && held > crate::CHUNK_BUDGET {
736 k.over_budget.fetch_add(1, Ordering::Relaxed);
737 }
738 self.longest.fetch_max(
739 (micros << MICROS_SHIFT) | kind.index() as u64,
740 Ordering::Relaxed,
741 );
742 }
743
744 /// A consistent-enough picture for a dashboard.
745 ///
746 /// Not a torn-read-free snapshot, and it does not pretend to be: the
747 /// actor keeps running while this walks the array, so two kinds may be
748 /// read one turn apart. Locking the actor to produce a report would make
749 /// the observer a source of the latency it is measuring.
750 pub fn snapshot(&self) -> super::MetricsSnapshot {
751 let samples = self.depth_samples.load(Ordering::Relaxed);
752 let mean = |sum: &AtomicU64| {
753 if samples == 0 {
754 0.0
755 } else {
756 sum.load(Ordering::Relaxed) as f64 / samples as f64
757 }
758 };
759
760 let packed = self.longest.load(Ordering::Relaxed);
761 let longest_micros = packed >> MICROS_SHIFT;
762 let longest = (longest_micros > 0)
763 .then(|| {
764 let idx = (packed & KIND_MASK) as usize;
765 CommandKind::ALL
766 .get(idx)
767 .map(|&kind| (kind, Duration::from_micros(longest_micros)))
768 })
769 .flatten();
770
771 let kinds: Vec<_> = CommandKind::ALL
772 .iter()
773 .map(|&kind| {
774 let k = &self.kinds[kind.index()];
775 let turns = k.turns.load(Ordering::Relaxed);
776 let total = k.total_micros.load(Ordering::Relaxed);
777 super::KindSnapshot {
778 kind,
779 turns,
780 over_budget: k.over_budget.load(Ordering::Relaxed),
781 mean: total
782 .checked_div(turns)
783 .map_or(Duration::ZERO, Duration::from_micros),
784 longest: Duration::from_micros(k.longest_micros.load(Ordering::Relaxed)),
785 buckets: std::array::from_fn(|i| k.buckets[i].load(Ordering::Relaxed)),
786 }
787 })
788 .collect();
789
790 super::MetricsSnapshot {
791 // Summed, not counted separately — see `depth_samples`.
792 turns: kinds.iter().map(|k| k.turns).sum(),
793 depth_samples: samples,
794 high_depth_mean: mean(&self.high_depth_sum),
795 high_depth_max: self.high_depth_max.load(Ordering::Relaxed),
796 low_depth_mean: mean(&self.low_depth_sum),
797 low_depth_max: self.low_depth_max.load(Ordering::Relaxed),
798 low_starved_turns: self.low_starved_turns.load(Ordering::Relaxed),
799 low_starved_run_max: self.low_starved_run_max.load(Ordering::Relaxed),
800 longest,
801 kinds,
802 }
803 }
804 }
805}
806
807// ---------------------------------------------------------------------------
808// No-op implementation
809// ---------------------------------------------------------------------------
810
811#[cfg(not(feature = "metrics"))]
812mod imp {
813 use super::CommandKind;
814 use std::time::Duration;
815
816 /// The `metrics`-off shape: zero-sized, and every method is nothing.
817 #[derive(Debug, Default)]
818 pub struct ActorMetrics;
819
820 impl ActorMetrics {
821 pub fn new() -> Self {
822 Self
823 }
824 #[inline]
825 pub fn record_turn(&self, _high_depth: usize, _low_depth: usize) {}
826 #[inline]
827 pub fn record_priority_choice(&self, _took_high: bool, _low_queued: usize) {}
828 #[inline]
829 pub fn record_hold(&self, _kind: CommandKind, _held: Duration) {}
830 }
831}
832
833pub use imp::ActorMetrics;
834
835/// Saturation point for a recorded hold, in microseconds (~2,000 years).
836///
837/// Exists so the packed `longest` field cannot have a pathological duration
838/// overflow into the kind bits. A hold this long is not a measurement, it is a
839/// hang — and the counter should stay readable rather than start reporting the
840/// wrong command.
841///
842/// Kept out of the `metrics` cfg so the invariant test below runs in the default
843/// build too: the packing is a property of the layout, and a build that does not
844/// record is exactly the build where nobody would notice it break.
845#[allow(dead_code)]
846const MICROS_CEILING: u64 = (1u64 << 56) - 1;
847
848/// One command kind's holds, as of the moment [`ActorMetrics::snapshot`] read it.
849#[cfg(feature = "metrics")]
850#[derive(Debug, Clone, PartialEq, Eq)]
851#[non_exhaustive]
852pub struct KindSnapshot {
853 pub kind: CommandKind,
854 /// Turns spent on this kind.
855 pub turns: u64,
856 /// Turns that exceeded [`crate::CHUNK_BUDGET`]. Always 0 for the kinds
857 /// [`CommandKind::exempt_from_budget`] names — see there for why, and for
858 /// the criterion that decides which those are.
859 ///
860 /// **Occurrences, not magnitude**: one per turn that exceeded, however far
861 /// it exceeded by. A kind whose hold has doubled reports the same count and
862 /// a different [`Self::longest`].
863 pub over_budget: u64,
864 pub mean: Duration,
865 /// This kind's longest hold. Distinct from [`MetricsSnapshot::longest`],
866 /// which names one command across all kinds and so tends to be permanently
867 /// whichever kind is slowest overall.
868 pub longest: Duration,
869 /// Counts per [`BUCKET_BOUNDS_MICROS`], plus a final overflow bucket.
870 ///
871 /// Private behind [`Self::buckets`] since 0.12.8 (W4.2). A public array
872 /// field publishes `BUCKET_COUNT` as part of the type's shape, so adding a
873 /// bucket bound would break every caller that named the length — and the
874 /// bounds are exactly the thing a latency histogram is likely to want to
875 /// re-cut. The accessor returns a slice and the length becomes an
876 /// observation rather than a signature. Python already did it this way.
877 buckets: [u64; BUCKET_COUNT],
878}
879
880#[cfg(feature = "metrics")]
881impl KindSnapshot {
882 /// Counts per [`BUCKET_BOUNDS_MICROS`], plus a final overflow bucket.
883 ///
884 /// Pair it with `BUCKET_BOUNDS_MICROS` to label the axis rather than
885 /// hard-coding the bounds; the slice is one longer than that constant,
886 /// and the extra trailing element is the overflow bucket.
887 pub fn buckets(&self) -> &[u64] {
888 &self.buckets
889 }
890}
891
892/// What the actor has done since the database was opened.
893#[cfg(feature = "metrics")]
894#[derive(Debug, Clone, PartialEq)]
895#[non_exhaustive]
896pub struct MetricsSnapshot {
897 /// Commands executed, i.e. the sum of [`KindSnapshot::turns`]. The two agree
898 /// by construction rather than by coincidence.
899 pub turns: u64,
900 /// Loop iterations that took a queue-depth reading. Always at least
901 /// `turns + 1` on a live actor, because the reading is taken on the way in
902 /// to a `select!` that has not resolved yet. This is the denominator of the
903 /// two means below, and it is exposed so the difference is visible rather
904 /// than looking like drift.
905 pub depth_samples: u64,
906 pub high_depth_mean: f64,
907 pub high_depth_max: u64,
908 pub low_depth_mean: f64,
909 pub low_depth_max: u64,
910 /// The longest hold since open and what caused it. `None` before the first
911 /// turn, and — honestly — also when every turn so far took under a
912 /// microsecond, which on this path does not happen.
913 pub longest: Option<(CommandKind, Duration)>,
914 /// Turns spent on high-priority work while low-priority work was already
915 /// queued (0.12.10, W4.4, D-153).
916 ///
917 /// The actor's `select!` is `biased` and has **no floor**, so this is the
918 /// measurement of a bound the design has always had and never observed.
919 /// On its own it is not alarming: a busy database *should* prefer
920 /// interactive writes, and this counter rising is that working. Read it
921 /// beside [`Self::low_starved_run_max`], which is the number with teeth.
922 pub low_starved_turns: u64,
923 /// The longest unbroken run of the above — i.e. the most turns any single
924 /// low-priority command has waited (0.12.10, W4.4, D-153).
925 ///
926 /// This is the one that answers "can low-priority work be starved". A large
927 /// `low_starved_turns` spread over a long session says the tiers are doing
928 /// their job; a large *run* says one specific chunk, rebuild or archive sat
929 /// behind that many interactive writes in a row.
930 ///
931 /// # There is deliberately no forced-yield policy, and the reason changed
932 /// (0.13.26, W10.4, [D-199])
933 ///
934 /// It used to be "adding one now would be fixing a bound nobody has
935 /// observed being hit". That premise died twice. [D-153] hit the bound
936 /// completely on a synthetic burst, and W10.4 then hit it on an ordinary
937 /// one: **four closed-loop writers** — each awaiting its own write before
938 /// issuing the next, which is what application code does — starve the low
939 /// tier for essentially all of their writes
940 /// (`examples/fairness_probe.rs`). The run is bounded by how long the
941 /// caller keeps offering interactive work, not by concurrency and not by
942 /// anything in this crate.
943 ///
944 /// **What replaced it is the floor's own price.** "After N starved turns,
945 /// take one low-priority command" cannot choose *which* command — the low
946 /// queue is an mpsc channel and its head is not inspectable — and at least
947 /// one low-priority kind is exempt from [`crate::CHUNK_BUDGET`] **by
948 /// contract**: an [`crate::Database::archive`] was measured at 3.3 s
949 /// unwindowed on an 8,000-key backlog. So the floor would add an unbounded
950 /// term to the interactive worst case in order to unblock work that is
951 /// declared not to be latency-sensitive, which is the tier split running
952 /// backwards.
953 ///
954 /// **The lever that does work belongs to the caller**: 1 ms of think time
955 /// between a writer's writes takes four writers from ~78 to ~2. Which makes
956 /// this field the instrument for a decision the caller owns rather than a
957 /// defect report about the actor.
958 ///
959 /// [D-153]: ../docs/architecture/s13-decision-register.md#d-153
960 /// [D-199]: ../docs/architecture/s13-decision-register.md#d-199
961 pub low_starved_run_max: u64,
962 pub kinds: Vec<KindSnapshot>,
963}
964
965#[cfg(feature = "metrics")]
966impl MetricsSnapshot {
967 /// Kinds that broke the budget, worst first. The one-line answer to "is the
968 /// 3 ms bound holding?".
969 pub fn budget_violations(&self) -> Vec<&KindSnapshot> {
970 let mut v: Vec<_> = self.kinds.iter().filter(|k| k.over_budget > 0).collect();
971 v.sort_by_key(|k| std::cmp::Reverse(k.over_budget));
972 v
973 }
974}
975
976#[cfg(test)]
977mod tests {
978 use super::*;
979
980 #[test]
981 fn every_kind_indexes_to_its_own_slot() {
982 for (i, &kind) in CommandKind::ALL.iter().enumerate() {
983 assert_eq!(kind.index(), i, "{kind} is out of order in ALL");
984 }
985 assert_eq!(CommandKind::COUNT, CommandKind::ALL.len());
986 }
987
988 /// The budget is a bucket boundary, not a value inside one — so "fits in the
989 /// budget" is a prefix sum and needs no interpolation.
990 #[test]
991 fn the_chunk_budget_is_exactly_a_bucket_boundary() {
992 let budget = crate::CHUNK_BUDGET.as_micros() as u64;
993 assert!(
994 BUCKET_BOUNDS_MICROS.contains(&budget),
995 "CHUNK_BUDGET is {budget} µs, which is not a bucket bound: \
996 {BUCKET_BOUNDS_MICROS:?}"
997 );
998 assert_eq!(bucket_of(budget), bucket_of(budget - 1));
999 assert_eq!(bucket_of(budget + 1), bucket_of(budget) + 1);
1000 }
1001
1002 #[test]
1003 fn the_overflow_bucket_catches_everything_past_the_last_bound() {
1004 let last = *BUCKET_BOUNDS_MICROS.last().unwrap();
1005 assert_eq!(bucket_of(last), BUCKET_BOUNDS_MICROS.len() - 1);
1006 assert_eq!(bucket_of(last + 1), BUCKET_COUNT - 1);
1007 assert_eq!(bucket_of(u64::MAX), BUCKET_COUNT - 1);
1008 }
1009
1010 /// The packing is the reason `longest` is one atomic: duration high, kind
1011 /// low, so a `fetch_max` on the word compares the duration.
1012 #[test]
1013 fn the_packing_leaves_room_for_both_fields() {
1014 assert!(
1015 (CommandKind::COUNT as u64) <= 0xFF,
1016 "the kind index must fit in the low 8 bits"
1017 );
1018 // The ceiling must survive being shifted up by the kind's width.
1019 assert_eq!(MICROS_CEILING.checked_shl(8), Some(MICROS_CEILING << 8));
1020 assert_eq!((MICROS_CEILING << 8) >> 8, MICROS_CEILING);
1021 }
1022
1023 /// The two halves of a shadow rebuild land on opposite sides of the
1024 /// budget, and a forged hold is the only way to assert it (0.14.16, D-233).
1025 ///
1026 /// The integration suite can run a real rebuild and check that the
1027 /// violation list comes back empty; what it cannot do is make a *fill*
1028 /// chunk run long on demand. So the canary lives here, where the hold is an
1029 /// argument: the same over-budget duration recorded against each half must
1030 /// produce a violation for one and not the other.
1031 ///
1032 /// Without this, widening the exemption to cover both halves would leave
1033 /// every test in the crate green — `a_swap_over_budget_is_not_a_violation`
1034 /// included, since it asserts a zero that a broader exemption also
1035 /// produces. This is the assertion that says the zero means *healthy* and
1036 /// not *unwatched*.
1037 #[cfg(feature = "metrics")]
1038 #[test]
1039 fn a_long_fill_is_a_violation_and_a_long_swap_is_not() {
1040 let m = ActorMetrics::new();
1041 let over = crate::CHUNK_BUDGET + Duration::from_millis(44);
1042
1043 m.record_hold(CommandKind::ShadowRebuild, over);
1044 m.record_hold(CommandKind::ShadowSwap, over);
1045
1046 let snap = m.snapshot();
1047 let of = |kind: CommandKind| {
1048 snap.kinds
1049 .iter()
1050 .find(|k| k.kind == kind)
1051 .unwrap()
1052 .over_budget
1053 };
1054
1055 assert_eq!(
1056 of(CommandKind::ShadowRebuild),
1057 1,
1058 "a fill chunk ran {over:?} against a {:?} budget and was not \
1059 counted. The fill half is the canary D-082 refused to exempt and \
1060 D-233 kept unexempted; if it stops counting, a regression on the \
1061 one path the chunked rebuild exists to keep short is invisible.",
1062 crate::CHUNK_BUDGET
1063 );
1064 assert_eq!(
1065 of(CommandKind::ShadowSwap),
1066 0,
1067 "the swap was counted as a violation. It exceeds by construction \
1068 on every healthy database, so counting it makes \
1069 `budget_violations()` nonzero forever (D-233)."
1070 );
1071
1072 // And the magnitude survives the exemption, which is the half of the
1073 // argument that decided C over B: exempting removes the *occurrence*
1074 // from the violation list and touches nothing a reader consults to see
1075 // the hold grow.
1076 let longest = snap
1077 .kinds
1078 .iter()
1079 .find(|k| k.kind == CommandKind::ShadowSwap)
1080 .unwrap()
1081 .longest;
1082 assert_eq!(
1083 longest, over,
1084 "the swap's hold stopped being recorded when it stopped being \
1085 counted. `over_budget` counts occurrences; the histogram and \
1086 `longest` are where growth is visible, and an exemption must not \
1087 reach them."
1088 );
1089 }
1090
1091 #[cfg(feature = "metrics")]
1092 #[test]
1093 fn the_longest_hold_names_the_command_that_caused_it() {
1094 let m = ActorMetrics::new();
1095 m.record_hold(CommandKind::AssertEdge, Duration::from_micros(500));
1096 m.record_hold(CommandKind::Archive, Duration::from_millis(40));
1097 m.record_hold(CommandKind::UpsertConcept, Duration::from_micros(900));
1098
1099 let snap = m.snapshot();
1100 assert_eq!(
1101 snap.longest,
1102 Some((CommandKind::Archive, Duration::from_millis(40)))
1103 );
1104 }
1105
1106 /// The regression the packing bug produced: a *short* hold of a
1107 /// later-declared kind must not outrank a long hold of an earlier one.
1108 ///
1109 /// The test above does not catch it, because `Archive` happens to be both
1110 /// the longest hold and a high enum index — which is exactly why the first
1111 /// version of the packing shipped past it. Here the two orderings disagree.
1112 #[cfg(feature = "metrics")]
1113 #[test]
1114 fn a_later_declared_kind_does_not_outrank_a_longer_hold() {
1115 let long = CommandKind::AssertEdge; // index 0
1116 let short = CommandKind::RebuildFts; // last index
1117 assert!(short.index() > long.index(), "the fixture needs the gap");
1118
1119 let m = ActorMetrics::new();
1120 m.record_hold(long, Duration::from_millis(40));
1121 m.record_hold(short, Duration::from_micros(1));
1122
1123 assert_eq!(
1124 m.snapshot().longest,
1125 Some((long, Duration::from_millis(40))),
1126 "the max is being taken over the kind index, not the duration"
1127 );
1128 }
1129
1130 /// The three contractual exemptions must not show up as violations, or the
1131 /// violation count is noise on any database that archives.
1132 #[cfg(feature = "metrics")]
1133 #[test]
1134 fn an_exempt_kind_over_budget_is_not_a_violation() {
1135 let m = ActorMetrics::new();
1136 m.record_hold(CommandKind::Archive, Duration::from_millis(40));
1137 m.record_hold(CommandKind::AssertEdge, Duration::from_millis(40));
1138
1139 let snap = m.snapshot();
1140 let violations = snap.budget_violations();
1141 assert_eq!(violations.len(), 1);
1142 assert_eq!(violations[0].kind, CommandKind::AssertEdge);
1143 assert_eq!(violations[0].over_budget, 1);
1144
1145 // But the hold is still *recorded* — exempt means "not a violation",
1146 // not "not measured". A 40 ms archive is exactly what T1.1 exists to
1147 // shrink, and it cannot be shrunk if it is not counted.
1148 let archive = snap
1149 .kinds
1150 .iter()
1151 .find(|k| k.kind == CommandKind::Archive)
1152 .unwrap();
1153 assert_eq!(archive.turns, 1);
1154 assert_eq!(archive.mean, Duration::from_millis(40));
1155 }
1156
1157 #[cfg(feature = "metrics")]
1158 #[test]
1159 fn queue_depth_is_a_mean_and_a_high_water_mark() {
1160 let m = ActorMetrics::new();
1161 m.record_turn(0, 4);
1162 m.record_turn(10, 0);
1163
1164 let snap = m.snapshot();
1165 // No command ran, so `turns` is 0 while `depth_samples` is 2. The two
1166 // counters are different facts and this is the case that shows it.
1167 assert_eq!(snap.turns, 0);
1168 assert_eq!(snap.depth_samples, 2);
1169 assert_eq!(snap.high_depth_mean, 5.0);
1170 assert_eq!(snap.high_depth_max, 10);
1171 assert_eq!(snap.low_depth_mean, 2.0);
1172 assert_eq!(snap.low_depth_max, 4);
1173 }
1174}