//! Operator annotation data table for [`super::operator_docs`] (issue #2426).
//!
//! Split out of `operator_docs.rs` to keep each file under the campsite source
//! target (#1116); this is pure data. The annotations are keyed by the
//! `catalog::*` name constants so they stay wired to the real catalog — the
//! fail-closed completeness/uniqueness checks live in `operator_docs.rs`.
use super::catalog::{self, attr};
use super::operator_docs::{MetricDoc, MetricKind};
// Read-PHASE timings + reader/WAL resource gauges (issue #1707): a SECOND
// annotation table in its own file, because this one was already at the campsite
// source target (#1116) and the table is append-only data.
#[path = "operator_docs_annotations_read_phase.rs"]
mod read_phase;
/// Every operator annotation table, in declaration order. The renderer walks the
/// UNION (and sorts by name), so a family lives in whichever file it was declared
/// in — `operator_docs::operator_metric_docs` never reads one table directly.
pub(super) const ANNOTATION_TABLES: &[&[MetricDoc]] = &[ANNOTATIONS, read_phase::ANNOTATIONS];
/// Every operator annotation across [`ANNOTATION_TABLES`].
pub(super) fn all_annotations() -> impl Iterator<Item = &'static MetricDoc> {
ANNOTATION_TABLES.iter().copied().flatten()
}
/// The operator annotation table (catalog declaration order; renderer sorts by name).
pub(super) const ANNOTATIONS: &[MetricDoc] = &[
MetricDoc {
name: catalog::READ_ROWS,
kind: MetricKind::Counter,
unit: catalog::unit::ROWS,
summary: "Rows a read DELIVERED to its consumer (climbs incrementally during a long Flight merge scan).",
attributes: &[attr::SSTABLE_FORMAT],
interpretation: "Steadily rising under load is healthy; flat while a scan is in flight suggests a stall. It counts DELIVERED rows, not every row decoded: an abandoned stream reports the rows the consumer polled plus those already enqueued, so a row the producer was mid-send with when the consumer went away is excluded. The series can therefore read marginally below what a producer decoded, but never above the rows a query could have returned.",
round_item: "—",
},
MetricDoc {
name: catalog::READ_BYTES,
kind: MetricKind::Counter,
unit: catalog::unit::BYTES,
summary: "Total Data.db bytes read (post-decompression), counted once per chunk decode (compressed or not).",
attributes: &[attr::COMPRESSION],
interpretation: "Track against read.rows to spot read amplification; a spike with flat rows means wide scans. The decompressed-chunk cache does NOT reduce it: the scan feed reads each chunk off disk BEFORE consulting that cache, so a warm re-scan counts the same bytes again (the cache saves decompression, not I/O). Counted for EVERY subsystem that reads an SSTable, compaction included, so a bytes spike with flat read.rows can also be a compaction running.",
round_item: "—",
},
MetricDoc {
name: catalog::READ_PARTITIONS,
kind: MetricKind::Counter,
unit: catalog::unit::PARTITIONS,
summary: "Partitions a read DELIVERED rows from on core read paths; partitions SCANNED on Flight's k-way merge arm.",
attributes: &[attr::SSTABLE_FORMAT],
interpretation: "Rising in step with read.rows is healthy; many partitions for few rows indicates a full scan. Mind the producer difference when comparing series: core paths derive partition boundaries from EMITTED row keys, so a partition whose every row is tombstone-shadowed or TTL-expired contributes ZERO, while Flight's merge arm counts it as scanned. The gap is exactly the fully-suppressed partitions, so an amplification dashboard mixing the two producers can under-count partitions on the core side.",
round_item: "—",
},
MetricDoc {
name: catalog::READ_DURATION,
kind: MetricKind::Histogram,
unit: catalog::unit::SECONDS,
summary: "Distribution of single read/scan operation durations, at the core query read boundaries (manager scans, point reads, streaming handles).",
attributes: &[attr::SSTABLE_FORMAT],
interpretation: "Watch p99; a growing tail is the read-latency alarm.",
round_item: "—",
},
MetricDoc {
name: catalog::READ_PARTITION_LOOKUP,
kind: MetricKind::Counter,
unit: catalog::unit::DIMENSIONLESS,
summary: "Partition point lookups attempted, tagged hit/miss so a dashboard computes the hit ratio from one series.",
attributes: &[attr::RESULT, attr::LOOKUP_ROUTE, attr::SSTABLE_FORMAT],
interpretation: "A healthy point-read workload is dominated by hits; a miss-heavy ratio means keys are absent or mis-routed.",
round_item: "—",
},
MetricDoc {
name: catalog::READ_PARTITION_ACCESS_DISTINCT_PARTITIONS,
kind: MetricKind::Counter,
unit: catalog::unit::PARTITIONS,
summary: "Distinct partitions per repeat-access bucket, incremented once per closed window of the default-OFF access-distribution probe (#2827) — the workload's concentration shape, with no per-key label.",
attributes: &[attr::REPEAT_BUCKET, attr::SIZE_SOURCE],
interpretation: "Absent unless an operator enabled CQLITE_PARTITION_ACCESS_PROBE. CUMULATIVE across windows — do not price a cache off a scrape of this after several windows have closed; use the in-process WindowSummary. Mass in bucket 1 is a uniform, cache-hostile pattern; mass in 9-16/17+ is a hot set. A non-zero size_source=unavailable share means the byte total beside it is incomplete and the procedure refuses the window.",
round_item: "—",
},
MetricDoc {
name: catalog::READ_PARTITION_ACCESS_ACCESSES,
kind: MetricKind::Counter,
unit: catalog::unit::DIMENSIONLESS,
summary: "Accesses attributable to each repeat-access bucket, incremented once per closed probe window (#2827) — the sum of the repeat counts of the partitions in that bucket.",
attributes: &[attr::REPEAT_BUCKET],
interpretation: "CUMULATIVE across windows, like its siblings. accesses minus distinct_partitions is the number of reads a cache holding that bucket could have served; equal values mean every access was a first touch (nothing to cache).",
round_item: "—",
},
MetricDoc {
name: catalog::READ_PARTITION_ACCESS_BYTES,
kind: MetricKind::Counter,
unit: catalog::unit::BYTES,
summary: "Distinct-partition on-disk bytes per repeat-access bucket, incremented once per closed probe window (#2827), measured as each partition's successor gap; a partition read ten times counts its bytes once.",
attributes: &[attr::REPEAT_BUCKET],
interpretation: "CUMULATIVE across windows. Uncompressed on-disk bytes, the correct input for a decoded-size multiplier; zero only when every access reported size_source=unavailable. Never an estimate.",
round_item: "—",
},
MetricDoc {
name: catalog::READ_PARTITION_ACCESS_SAMPLE_DENOMINATOR,
kind: MetricKind::Gauge,
unit: catalog::unit::DIMENSIONLESS,
summary: "Sampling scale in force when the probe window closed (#2827): 1 is a census, 2^k means the recorder downsampled k times to stay inside its fixed 3 MiB table.",
attributes: &[],
interpretation: "1 is ideal. A large value means the window touched far more distinct partitions than the table holds; bucket fractions stay unbiased but absolute counts must be scaled by it.",
round_item: "—",
},
MetricDoc {
name: catalog::READ_PARTITION_ACCESS_DROPPED,
kind: MetricKind::Counter,
unit: catalog::unit::DIMENSIONLESS,
summary: "Accesses the access-distribution probe could not seat in its fixed counting table (#2827).",
attributes: &[],
interpretation: "Zero on a healthy window. Non-zero INVALIDATES the window: only keys not already in the table can be dropped, so a loss suppresses the singleton bucket and overstates concentration — the cache-sizing procedure refuses such a window.",
round_item: "—",
},
MetricDoc {
name: catalog::READ_PARTITION_ACCESS_SAMPLING_FLOOR,
kind: MetricKind::Gauge,
unit: catalog::unit::DIMENSIONLESS,
summary: "1 when the last closed probe window reached its sampling-prefix cap, 0 otherwise (#2827).",
attributes: &[],
interpretation: "0 is healthy. 1 means the window is a ~1-in-a-million sample and is statistically worthless; the cache-sizing procedure refuses it. Read alongside sample_denominator.",
round_item: "—",
},
MetricDoc {
name: catalog::READ_PARTITION_ACCESS_WINDOW_DROPPED,
kind: MetricKind::Gauge,
unit: catalog::unit::DIMENSIONLESS,
summary: "Accesses the LAST CLOSED probe window could not seat, reset every window (#2827).",
attributes: &[],
interpretation: "A window is CLEAN exactly when this and sampling_floor both read 0. Use this, not the cumulative dropped_accesses counter, to judge the current window: a counter that ever incremented reads non-zero for the life of the process.",
round_item: "—",
},
MetricDoc {
name: catalog::READ_BLOOM_CHECKS,
kind: MetricKind::Counter,
unit: catalog::unit::DIMENSIONLESS,
summary: "Bloom-filter / BTI-trie present-or-absent checks; miss = definitely absent.",
attributes: &[attr::RESULT, attr::SSTABLE_FORMAT],
interpretation: "A high miss-rate is healthy pruning; pairing with partition_lookup reveals the bloom false-positive rate.",
round_item: "—",
},
MetricDoc {
name: catalog::READ_SCAN_WINDOW_REFILL,
kind: MetricKind::Counter,
unit: catalog::unit::DIMENSIONLESS,
summary: "Windowed streaming-scan refills at a compression-chunk boundary (a partition straddles the chunk edge and the driver awaits the next decompressed chunk).",
attributes: &[],
interpretation: "Non-zero proves the multi-chunk stitch path was exercised; it stays 0 for single-chunk SSTables. A runaway value means excessive chunk-boundary straddling.",
round_item: "—",
},
MetricDoc {
name: catalog::READ_SSTABLES_PRUNED,
kind: MetricKind::Counter,
unit: catalog::unit::SSTABLES,
summary: "SSTables skipped by a definitive presence-oracle negative (bloom/BTI-trie miss).",
attributes: &[attr::SSTABLE_FORMAT],
interpretation: "Higher is better — it is the dashboard-honest prune signal; zero on a point read means no pruning happened.",
round_item: "—",
},
MetricDoc {
name: catalog::READ_BTI_ROWS_ROOT_REJECTED,
kind: MetricKind::Counter,
unit: catalog::unit::PARTITIONS,
summary: "Clustering-slice reads that decoded a WHOLE BTI partition because its Rows.db row-index root failed structural validation, tagged by the violated invariant.",
attributes: &[attr::ROWS_ROOT_REJECT_REASON],
interpretation: "Should be 0. Non-zero explains clustering-read latency with no narrowing: the rows are correct but the row index is unusable — re-flush/re-compact a Rows.db written by CQLite <= 0.16 (#3002).",
round_item: "—",
},
MetricDoc {
name: catalog::READ_BLOOM_FALSE_NEGATIVES,
kind: MetricKind::Counter,
unit: catalog::unit::DIMENSIONLESS,
summary: "Opt-in soundness alarm: keys found on an authoritative scan that the presence oracle said were absent.",
attributes: &[attr::SSTABLE_FORMAT],
interpretation: "MUST stay 0. Any non-zero value is a corruption/soundness alarm (only emitted when verification is enabled).",
round_item: "—",
},
MetricDoc {
name: catalog::MERGE_ROWS_IN,
kind: MetricKind::Counter,
unit: catalog::unit::ROWS,
summary: "Input rows consumed at the k-way merge reconcile boundary (once per merge).",
attributes: &[],
interpretation: "Compare with merge.rows_out; the gap is rows removed by last-write-wins collapse and tombstone suppression.",
round_item: "—",
},
MetricDoc {
name: catalog::MERGE_ROWS_OUT,
kind: MetricKind::Counter,
unit: catalog::unit::ROWS,
summary: "Rows emitted by the merge reconcile boundary post-reconciliation.",
attributes: &[],
interpretation: "See merge.rows_in; rows_out much smaller than rows_in means heavy reconciliation.",
round_item: "—",
},
MetricDoc {
name: catalog::QUERY_DEGRADED_PATH,
kind: MetricKind::Counter,
unit: catalog::unit::DIMENSIONLESS,
summary: "SELECTs that took a soundness fallback to a full-scan read path, tagged by fallback reason.",
attributes: &[attr::FALLBACK_REASON],
interpretation: "Should stay near 0 for targeted queries; a climbing value means queries are silently degrading to full scans.",
round_item: "—",
},
MetricDoc {
name: catalog::INDEX_PARSES_TOTAL,
kind: MetricKind::Counter,
unit: catalog::unit::DIMENSIONLESS,
summary: "Whole-file BIG/NB Index.db parses; the scale-free probe for the #2383 resolve-phase CPU spin.",
attributes: &[],
interpretation: "Healthy ≈ sum of generations per query; climbing far past that means redundant re-parses (a #2383-class regression).",
round_item: "index-parse audit (#2367/#2399)",
},
MetricDoc {
name: catalog::INDEX_INTERVAL_PARSES_TOTAL,
kind: MetricKind::Counter,
unit: catalog::unit::DIMENSIONLESS,
summary: "Bounded Summary-guided Index.db interval parses (one per point lookup); distinct from whole-file parses.",
attributes: &[],
interpretation: "Grows ~1 per point lookup; a cold lazy open adds 0. Whole-file spikes stay visible on index_parses_total.",
round_item: "index-parse audit (#2367/#2399)",
},
MetricDoc {
name: catalog::KEY_CACHE_HITS,
kind: MetricKind::Counter,
unit: catalog::unit::DIMENSIONLESS,
summary: "Surfaced only via `Database::stats().memory_stats` — NOT emitted as a live OTel instrument (not scrapeable from Prometheus/an OTel collector). Hits on the process-global key→partition-offset cache (a point read skips the Index.db interval parse / trie descent).",
attributes: &[],
interpretation: "A high hit rate means hot partitions are resolving from cache; near-zero on a cold or highly-random point-read workload.",
round_item: "read-path perf (#2059)",
},
MetricDoc {
name: catalog::KEY_CACHE_MISSES,
kind: MetricKind::Counter,
unit: catalog::unit::DIMENSIONLESS,
summary: "Surfaced only via `Database::stats().memory_stats` — NOT emitted as a live OTel instrument (not scrapeable from Prometheus/an OTel collector). Misses on the global key cache (incl. fail-closed identity mismatch); each pays one interval parse / trie descent then populates.",
attributes: &[],
interpretation: "Rises with working-set churn or eviction pressure; a persistently high miss ratio suggests the budget is small vs the hot set.",
round_item: "read-path perf (#2059)",
},
MetricDoc {
name: catalog::KEY_CACHE_EVICTIONS,
kind: MetricKind::Counter,
unit: catalog::unit::DIMENSIONLESS,
summary: "Surfaced only via `Database::stats().memory_stats` — NOT emitted as a live OTel instrument (not scrapeable from Prometheus/an OTel collector). Global key-cache entries evicted to stay within the byte budget (budget-driven; distinct from invalidations).",
attributes: &[],
interpretation: "Sustained growth means the hot location set exceeds the fixed budget; expected to be flat on a small working set.",
round_item: "read-path perf (#2059)",
},
MetricDoc {
name: catalog::KEY_CACHE_INVALIDATIONS,
kind: MetricKind::Counter,
unit: catalog::unit::DIMENSIONLESS,
summary: "Surfaced only via `Database::stats().memory_stats` — NOT emitted as a live OTel instrument (not scrapeable from Prometheus/an OTel collector). Global key-cache entries dropped on generation removal/compaction/warm-evict (distinct from budget evictions).",
attributes: &[],
interpretation: "Tracks generation turnover (compaction/refresh); a #2383 rebind over a byte-identical generation does NOT invalidate.",
round_item: "read-path perf (#2059)",
},
MetricDoc {
name: catalog::KEY_CACHE_RESIDENT_BYTES,
kind: MetricKind::Gauge,
unit: catalog::unit::BYTES,
summary: "Surfaced only via `Database::stats().memory_stats` — NOT emitted as a live OTel instrument (not scrapeable from Prometheus/an OTel collector). Approximate resident footprint of the process-global key cache.",
attributes: &[],
interpretation: "Bounded above by capacity_bytes; a single global cap regardless of how many readers are open.",
round_item: "read-path perf (#2059)",
},
MetricDoc {
name: catalog::KEY_CACHE_CAPACITY_BYTES,
kind: MetricKind::Gauge,
unit: catalog::unit::BYTES,
summary: "Surfaced only via `Database::stats().memory_stats` — NOT emitted as a live OTel instrument (not scrapeable from Prometheus/an OTel collector). The global key cache's fixed byte budget (0 when block caching is disabled).",
attributes: &[],
interpretation: "A fixed named constant inside the <128MB envelope; not a user knob. Zero means the read caches are disabled.",
round_item: "read-path perf (#2059)",
},
MetricDoc {
name: catalog::STORAGE_OPEN_SSTABLES,
kind: MetricKind::Counter,
unit: catalog::unit::SSTABLES,
summary: "SSTables discovered and opened per StorageEngine open, summed over process lifetime.",
attributes: &[],
interpretation: "Tracks how much on-disk state each open touches; sudden growth signals compaction lag or fan-out.",
round_item: "—",
},
MetricDoc {
name: catalog::STORAGE_OPEN_BYTES,
kind: MetricKind::Counter,
unit: catalog::unit::BYTES,
summary: "Total on-disk Data.db bytes across the SSTables discovered at open.",
attributes: &[],
interpretation: "The dataset size an open must account for; use with open.sstables to size cold-start cost.",
round_item: "—",
},
MetricDoc {
name: catalog::STORAGE_OPEN_TABLES,
kind: MetricKind::Counter,
unit: catalog::unit::DIMENSIONLESS,
summary: "Logical tables represented by the SSTables discovered at open.",
attributes: &[],
interpretation: "Baseline inventory count; unexpected changes mean schema/keyspace drift on disk.",
round_item: "—",
},
MetricDoc {
name: catalog::QUERY_DURATION,
kind: MetricKind::Histogram,
unit: catalog::unit::SECONDS,
summary: "End-to-end query execution durations.",
attributes: &[attr::SUBSYSTEM],
interpretation: "Watch p99; a rising tail is the top-level query-latency alarm.",
round_item: "—",
},
MetricDoc {
name: catalog::QUERY_ROWS,
kind: MetricKind::Counter,
unit: catalog::unit::ROWS,
summary: "Rows returned to callers by the query engine.",
attributes: &[attr::ACCESS_PATH, attr::PLAN_TYPE],
interpretation: "Compare with query.rows_scanned; a large gap is read amplification.",
round_item: "—",
},
MetricDoc {
name: catalog::QUERY_ROWS_SCANNED,
kind: MetricKind::Counter,
unit: catalog::unit::ROWS,
summary: "Rows examined by the SELECT scan step before filter/projection/LIMIT.",
attributes: &[attr::ACCESS_PATH],
interpretation: "≈ query.rows for a point lookup; far larger means a full_scan is doing the work.",
round_item: "—",
},
MetricDoc {
name: catalog::SSTABLES_OPEN,
kind: MetricKind::Gauge,
unit: catalog::unit::SSTABLES,
summary: "SSTables currently held open.",
attributes: &[attr::SSTABLE_FORMAT],
interpretation: "A stable level is healthy; unbounded growth is a handle/file-descriptor leak.",
round_item: "—",
},
MetricDoc {
name: catalog::COMPACTION_DURATION,
kind: MetricKind::Histogram,
unit: catalog::unit::SECONDS,
summary: "Compaction run durations.",
attributes: &[],
interpretation: "Use with compaction.rows_merged for merge throughput; a growing tail means compaction is falling behind.",
round_item: "—",
},
MetricDoc {
name: catalog::ERRORS_TOTAL,
kind: MetricKind::Counter,
unit: catalog::unit::ERRORS,
summary: "Canonical error-rate signal, keyed by bounded {category, subsystem}; flight do_get aborts also carry flight.abort_reason; eagerly registered at 0 on startup.",
attributes: &[attr::ERROR_CATEGORY, attr::SUBSYSTEM, attr::FLIGHT_ABORT_REASON],
interpretation: "Absent metric name = error counting isn't wired (never 'no errors'); any sustained increase is the error-rate alarm; on subsystem=flight, split abort_reason to separate benign aborts (client_cancel/superseded_split/snapshot_retired/admission_shed) from genuine internal faults.",
round_item: "error-rate watch (#2193/#2399/#2681)",
},
MetricDoc {
name: catalog::WRITE_MUTATIONS,
kind: MetricKind::Counter,
unit: catalog::unit::ROWS,
summary: "Mutations accepted by the write path (one per successful memtable insert).",
attributes: &[],
interpretation: "Tracks write throughput; flatlining under offered load means writes are blocked.",
round_item: "—",
},
MetricDoc {
name: catalog::MEMTABLE_SIZE_BYTES,
kind: MetricKind::Gauge,
unit: catalog::unit::BYTES,
summary: "Current approximate in-memory size of the active memtable.",
attributes: &[],
interpretation: "Sawtooth (rise then drop at flush) is healthy; a monotonic climb means flush is not keeping up.",
round_item: "—",
},
MetricDoc {
name: catalog::MEMTABLE_ROWS,
kind: MetricKind::Gauge,
unit: catalog::unit::ROWS,
summary: "Buffered rows in the active memtable.",
attributes: &[],
interpretation: "Same sawtooth shape as memtable.size_bytes; a rising floor signals flush pressure.",
round_item: "—",
},
MetricDoc {
name: catalog::WAL_SYNC_DURATION,
kind: MetricKind::Histogram,
unit: catalog::unit::SECONDS,
summary: "WAL fsync durations.",
attributes: &[],
interpretation: "A growing tail points at slow/contended disk — the write-durability latency alarm.",
round_item: "—",
},
MetricDoc {
name: catalog::FLUSH_DURATION,
kind: MetricKind::Histogram,
unit: catalog::unit::SECONDS,
summary: "Memtable-to-SSTable flush durations.",
attributes: &[],
interpretation: "A rising tail alongside a climbing memtable size means flush cannot drain the memtable fast enough.",
round_item: "—",
},
MetricDoc {
name: catalog::FLUSH_ROWS,
kind: MetricKind::Counter,
unit: catalog::unit::ROWS,
summary: "Rows flushed from the memtable to L0 SSTables.",
attributes: &[],
interpretation: "Should track write.mutations over time; a lag is the flush-behind signal.",
round_item: "—",
},
MetricDoc {
name: catalog::FLUSH_BYTES,
kind: MetricKind::Counter,
unit: catalog::unit::BYTES,
summary: "Data.db bytes produced by memtable flushes.",
attributes: &[],
interpretation: "Write volume landing on disk from flush; use with flush.duration for flush throughput.",
round_item: "—",
},
MetricDoc {
name: catalog::FLUSH_SSTABLES,
kind: MetricKind::Counter,
unit: catalog::unit::SSTABLES,
summary: "L0 SSTables created by memtable flushes.",
attributes: &[],
interpretation: "Feeds compaction.lag; a high rate with rising lag means L0 is accumulating faster than compaction clears it.",
round_item: "—",
},
MetricDoc {
name: catalog::WRITE_PARTITIONS,
kind: MetricKind::Counter,
unit: catalog::unit::PARTITIONS,
summary: "Partitions written by the SSTable writer (flush + compaction output).",
attributes: &[],
interpretation: "Baseline write-shape signal; interpret alongside write.bytes.",
round_item: "—",
},
MetricDoc {
name: catalog::WRITE_BYTES,
kind: MetricKind::Counter,
unit: catalog::unit::BYTES,
summary: "Data.db bytes produced by the SSTable writer across all outputs.",
attributes: &[],
interpretation: "Total on-disk write volume; use with compaction.bytes_written to reason about write amplification.",
round_item: "—",
},
MetricDoc {
name: catalog::COMPRESSION_RATIO,
kind: MetricKind::Histogram,
unit: catalog::unit::DIMENSIONLESS,
summary: "WRITE-SIDE ONLY: per-chunk compression ratio recorded by the SSTable writer as it compresses each chunk (compressed/uncompressed; <=1.0 means the chunk shrank). There is no read-side emission, so it says nothing about the SSTables being read.",
attributes: &[attr::COMPRESSION],
interpretation: "Ratios near 1.0 mean incompressible data; a sudden rise means less benefit from the configured codec. Silent unless compressed writing is in use (#1406: the production write surface emits uncompressed SSTables).",
round_item: "—",
},
MetricDoc {
name: catalog::COMPACTION_ROWS_MERGED,
kind: MetricKind::Counter,
unit: catalog::unit::ROWS,
summary: "Rows emitted by the k-way merge across all compactions.",
attributes: &[],
interpretation: "With compaction.duration this yields rows-merged-per-second; a drop signals compaction stalling.",
round_item: "—",
},
MetricDoc {
name: catalog::COMPACTION_BYTES_WRITTEN,
kind: MetricKind::Counter,
unit: catalog::unit::BYTES,
summary: "Bytes written to compaction output SSTables.",
attributes: &[],
interpretation: "Compare with write.bytes/flush.bytes to reason about write amplification from compaction.",
round_item: "—",
},
MetricDoc {
name: catalog::COMPACTION_SSTABLES_IN,
kind: MetricKind::Counter,
unit: catalog::unit::SSTABLES,
summary: "Input SSTables consumed by compactions.",
attributes: &[],
interpretation: "In vs out ratio is the consolidation factor; in >> out is healthy consolidation.",
round_item: "—",
},
MetricDoc {
name: catalog::COMPACTION_SSTABLES_OUT,
kind: MetricKind::Counter,
unit: catalog::unit::SSTABLES,
summary: "Output SSTables produced by compactions.",
attributes: &[],
interpretation: "See compaction.sstables_in; out approaching in means little consolidation is happening.",
round_item: "—",
},
MetricDoc {
name: catalog::COMPACTION_TOMBSTONES_PURGED,
kind: MetricKind::Counter,
unit: catalog::unit::TOMBSTONES,
summary: "Tombstones genuinely purged (gc-grace/overlap-safe) during compaction.",
attributes: &[],
interpretation: "Rising means space is being reclaimed; zero while tombstones accumulate means purge is blocked (gc_grace/overlap).",
round_item: "—",
},
MetricDoc {
name: catalog::COMPACTION_TOMBSTONES_SUPPRESSED,
kind: MetricKind::Counter,
unit: catalog::unit::TOMBSTONES,
summary: "Live cells/rows shadowed by a tombstone during merge reconciliation.",
attributes: &[],
interpretation: "Suppression without a matching purge or retained marker is the resurrection-risk smell.",
round_item: "—",
},
MetricDoc {
name: catalog::COMPACTION_TOMBSTONES_EMITTED,
kind: MetricKind::Counter,
unit: catalog::unit::TOMBSTONES,
summary: "Tombstone markers retained (carried forward, not purgeable) into merge output.",
attributes: &[],
interpretation: "Expected while data is within gc_grace; a persistent climb means tombstones are not aging out.",
round_item: "—",
},
MetricDoc {
name: catalog::COMPACTION_LAG,
kind: MetricKind::Gauge,
unit: catalog::unit::SSTABLES,
summary: "L0 SSTables currently pending compaction (compaction lag).",
attributes: &[],
interpretation: "A stable low level is healthy; a monotonic rise means compaction cannot keep up with flush.",
round_item: "compaction-lag watch (#2399)",
},
MetricDoc {
name: catalog::COMPACTION_FINALIZE_DURATION,
kind: MetricKind::Histogram,
unit: catalog::unit::SECONDS,
summary: "Compaction finalize (atomic rename / publication-barrier) durations.",
attributes: &[],
interpretation: "Normally small; a growing tail points at slow filesystem rename/barrier operations.",
round_item: "—",
},
MetricDoc {
name: catalog::COMPACTION_BUDGET_REQUESTED,
kind: MetricKind::Histogram,
unit: catalog::unit::SECONDS,
summary: "Maintenance budget requested per maintenance_step call.",
attributes: &[],
interpretation: "Compare against budget.consumed to confirm the scheduler honors its ~10% tolerance.",
round_item: "—",
},
MetricDoc {
name: catalog::COMPACTION_BUDGET_CONSUMED,
kind: MetricKind::Histogram,
unit: catalog::unit::SECONDS,
summary: "Maintenance budget actually consumed per maintenance_step call.",
attributes: &[],
interpretation: "Consumed materially exceeding requested means maintenance is overrunning its budget.",
round_item: "—",
},
MetricDoc {
name: catalog::MERGE_PRODUCER_THREADS,
kind: MetricKind::Gauge,
unit: catalog::unit::THREADS,
summary: "Live OS producer threads the k-way merge currently holds (one per input SSTable).",
attributes: &[],
interpretation: "Bounded by O(M) inputs and returns to baseline at merge completion; a stuck-high value is a thread leak (#2316).",
round_item: "thread-budget watch (#2313/#2399)",
},
MetricDoc {
name: catalog::MERGE_ACTIVE_MERGES,
kind: MetricKind::Gauge,
unit: catalog::unit::MERGES,
summary: "Live concurrent k-way merges — the divisor of the adaptive egress budget (#2765).",
attributes: &[],
interpretation: "Per-channel capacity is clamp(EGRESS_ROW_BUDGET/active, MIN_CAP, 256); a level well above budget/256 means concurrent merges are being throttled toward MIN_CAP.",
round_item: "egress-backpressure watch (#2765/#2367)",
},
MetricDoc {
name: catalog::RPC_REQUESTS,
kind: MetricKind::Counter,
unit: catalog::unit::DIMENSIONLESS,
summary: "Arrow Flight RPC requests served, tagged by method + ok/error.",
attributes: &[attr::RPC_METHOD, attr::RPC_STATUS],
interpretation: "Compute per-method error rate from the status arm; a rising error fraction is the RPC-health alarm.",
round_item: "rpc-error-rate watch (#2399)",
},
MetricDoc {
name: catalog::RPC_DURATION,
kind: MetricKind::Histogram,
unit: catalog::unit::SECONDS,
summary: "Arrow Flight RPC handler durations (includes admission wait time).",
attributes: &[attr::RPC_METHOD, attr::RPC_STATUS],
interpretation: "Watch do_get p99; localize a rising tail with rpc.phase.duration.",
round_item: "do_get latency (#2367/#2399)",
},
MetricDoc {
name: catalog::RPC_IN_FLIGHT,
kind: MetricKind::Gauge,
unit: catalog::unit::DIMENSIONLESS,
summary: "Arrow Flight RPCs currently being handled.",
attributes: &[attr::RPC_METHOD],
interpretation: "A sustained-high value with flat rpc.rows is a stall; distinct from admission.in_use (all RPCs, not just admitted scans).",
round_item: "—",
},
MetricDoc {
name: catalog::RPC_ROWS,
kind: MetricKind::Counter,
unit: catalog::unit::ROWS,
summary: "Rows returned to clients by do_get (emitted per record batch).",
attributes: &[attr::RPC_METHOD],
interpretation: "Climbing = healthy long scan; flat while rpc.in_flight>0 = a stalled do_get.",
round_item: "do_get progress (#2367/#2399)",
},
MetricDoc {
name: catalog::RPC_BYTES,
kind: MetricKind::Counter,
unit: catalog::unit::BYTES,
summary: "Record-batch payload bytes streamed to clients by do_get (pre-IPC-framing).",
attributes: &[attr::RPC_METHOD],
interpretation: "Tracks egress volume; pair with rpc.rows to see batch sizing.",
round_item: "—",
},
MetricDoc {
name: catalog::RPC_PHASE_DURATION,
kind: MetricKind::Histogram,
unit: catalog::unit::SECONDS,
summary: "Per-phase do_get wall time: the top-level validate/admission/resolve/merge_setup/stream, plus (within stream, do_get only) the in-stream sub-phases stream_cold_fault/stream_decompress/stream_merge/stream_encode/stream_encode_framing/stream_grpc_write.",
attributes: &[attr::RPC_METHOD, attr::RPC_PHASE],
interpretation: "Localizes WHERE a slow do_get spent time: piling up in merge_setup, queued in admission, or stuck parsing in validate. Within stream the sub-phases run on concurrent pipeline threads and OVERLAP (they do NOT sum to stream) — read the cold-warm delta on stream_cold_fault as the cold-IO latency bucket; stream_merge is merge CPU ONLY (the blocking merge-input recv wait is subtracted, so producer starvation does not inflate it); stream_grpc_write is CLIENT-PACED (egress park/wake), not server cost. stream_encode covers, on the merge-consumer thread, the Arrow array build AND (since issue #3552) the push-time cell resolution, per-row width charge and column-major commit that were folded into the row loop -- so it is NOT COMPARABLE ACROSS #3552 IN EITHER DIRECTION: it gained that push-time work and simultaneously lost the flush-time transpose, which moved INTO it. Neither stream_encode NOR merge+encode is comparable across that boundary -- the per-row width charge sat in NO sub-phase before #3552 and is inside stream_encode now, so the SUM gains it too. Use throughput (rows/s) or a per-row-normalised measurement instead. While stream_encode_framing (issue #3096) covers the arrow-flight encoder stage on the async gRPC task -- IPC serialization, dictionary hydration and encoder-target re-slicing -- so a framing change is attributable separately from an array-build change; stream_encode_framing is poll time on that task, so it also contains the cheap non-blocking channel poll that returns Pending while the merge has produced nothing yet. CAVEAT: stream_cold_fault/stream_decompress are SUMMED across the N concurrent per-SSTable scan threads, so on a multi-generation table they can EXCEED the stream phase duration (compare the cold-warm DELTA, never the absolute vs stream); and they are recorded only on the instrumented scan read paths (Summary-guided + full-ring fallback), so a near-zero value can mean 'this route was not measured', not 'cold IO negligible'. The six stream_* sub-phases are emitted only on the STREAMING do_get route; an aggregating do_get (materialized per-group output) emits none of them.",
round_item: "do_get phase + in-stream sub-phase breakdown (#2398/#2399/#2819/#3096)",
},
MetricDoc {
name: catalog::RPC_PHASE_ACTIVE,
kind: MetricKind::Gauge,
unit: catalog::unit::DIMENSIONLESS,
summary: "In-flight phase a do_get is CURRENTLY in (1 on entry, 0 on exit) — visible before the phase completes.",
attributes: &[attr::RPC_METHOD, attr::RPC_PHASE],
interpretation: "stream=1 for a whole hang exposes a wedged do_get that phase.duration would never record; admission=1 exposes an admission queue wait.",
round_item: "do_get hang detection (#2361/#2399)",
},
MetricDoc {
name: catalog::WARM_CACHE_HITS,
kind: MetricKind::Counter,
unit: catalog::unit::DIMENSIONLESS,
summary: "Flight warm-handle cache hits served from warm parsed state with zero reader-open.",
attributes: &[],
interpretation: "A high hit ratio (hits vs misses) is the warm-cache win; a low ratio means generations keep changing.",
round_item: "warm-cache hit ratio (#2310/#2399)",
},
MetricDoc {
name: catalog::WARM_CACHE_MISSES,
kind: MetricKind::Counter,
unit: catalog::unit::DIMENSIONLESS,
summary: "Flight warm-handle cache misses (no cached entry, or generation set changed).",
attributes: &[],
interpretation: "Pair with warm.cache.hits for the hit ratio; a rising miss rate erodes the warm-open speedup.",
round_item: "warm-cache hit ratio (#2310/#2399)",
},
MetricDoc {
name: catalog::WARM_CACHE_EVICTS,
kind: MetricKind::Counter,
unit: catalog::unit::DIMENSIONLESS,
summary: "Warm generations evicted by LRU byte-budget pressure or removal on disk.",
attributes: &[],
interpretation: "A high evict rate with a low hit ratio means the warm byte budget is too small for the working set.",
round_item: "—",
},
MetricDoc {
name: catalog::WARM_CACHE_REFRESH,
kind: MetricKind::Counter,
unit: catalog::unit::DIMENSIONLESS,
summary: "Warm-handle refresh outcomes, tagged unchanged/rebuilt_delta/fail_closed_retained.",
attributes: &[attr::WARM_REFRESH_OUTCOME],
interpretation: "Mostly unchanged/rebuilt_delta is healthy; a rising fail_closed_retained arm means refreshes are failing and serving stale-but-safe state.",
round_item: "—",
},
MetricDoc {
name: catalog::FLIGHT_ADMISSION_LIMIT,
kind: MetricKind::Gauge,
unit: catalog::unit::DIMENSIONLESS,
summary: "The configured do_get admission ceiling K (--max-concurrent-scans); constant while the server runs.",
attributes: &[],
interpretation: "Chart admission.in_use against this ceiling; in_use pinned at the limit means the server is saturated.",
round_item: "admission saturation (#2420/#2399)",
},
MetricDoc {
name: catalog::FLIGHT_ADMISSION_IN_USE,
kind: MetricKind::Gauge,
unit: catalog::unit::DIMENSIONLESS,
summary: "do_get admission permits currently held (admitted, in-flight scans only).",
attributes: &[],
interpretation: "Below the limit is healthy headroom; sitting at admission.limit with a non-zero waiting gauge is saturation.",
round_item: "admission saturation (#2420/#2399)",
},
MetricDoc {
name: catalog::FLIGHT_ADMISSION_WAITING,
kind: MetricKind::Gauge,
unit: catalog::unit::DIMENSIONLESS,
summary: "do_get requests parked waiting for an admission permit — the backpressure signal.",
attributes: &[],
interpretation: "Zero is healthy; a sustained non-zero value means offered concurrency exceeds the ceiling and requests are queuing.",
round_item: "admission saturation (#2420/#2399)",
},
MetricDoc {
name: catalog::FLIGHT_ADMISSION_REJECTED_TOTAL,
kind: MetricKind::Counter,
unit: catalog::unit::DIMENSIONLESS,
summary: "do_get requests rejected (gRPC UNAVAILABLE) because no permit freed within the wait timeout.",
attributes: &[],
interpretation: "Should stay 0 under normal load; any sustained increase means clients are being shed and should fail over.",
round_item: "admission saturation (#2420/#2399)",
},
MetricDoc {
name: catalog::FLIGHT_ADMISSION_WAIT_SECONDS,
kind: MetricKind::Histogram,
unit: catalog::unit::SECONDS,
summary: "How long a do_get waited on acquire before it was admitted or rejected.",
attributes: &[],
interpretation: "A near-zero distribution is healthy; a rising tail means requests are increasingly queuing for permits.",
round_item: "admission saturation (#2420/#2399)",
},
// Saturation instrumentation (#2419, WS2 of epic #2313).
MetricDoc {
name: catalog::MERGE_EGRESS_CHANNEL_DEPTH,
kind: MetricKind::Gauge,
unit: catalog::unit::ENTRIES,
summary: "Live occupancy, in ENTRIES (rows), of the bounded merge egress sync_channel feeding do_get / compaction — batched since #2820 (up to 256 rows per message). Its per-source ceiling is CHANNEL-RESIDENT rows, 2 x rows_cap = 512 at the default #2765 budget — NOT the 4 x rows_cap = 1024 total-in-flight MEMORY bound, which also counts the consumer-held and producer-blocked batches this gauge never sees.",
attributes: &[],
interpretation: "Near zero = consumer keeping up (or a stalled producer); riding near the per-source channel-resident bound (512 at the default) = producer outrunning a slower consumer (back-pressured egress). Threshold alerts on 2 x rows_cap, never on 1024: a 1024 threshold is UNREACHABLE per source and can never fire. Process-globally the gauge is the SUM over concurrent sources, so it can exceed any single source ceiling. The unit is ENTRIES, so a batch of n rows moves it by n, never by 1.",
round_item: "egress backpressure watch (#2419/#2399)",
},
MetricDoc {
name: catalog::PROC_THREADS,
kind: MetricKind::Gauge,
unit: catalog::unit::THREADS,
summary: "Process OS thread count sampled from /proc/self/task (Linux; absent off-/proc, never 0).",
attributes: &[],
interpretation: "Rises with concurrent scans and settles to baseline; a level climbing toward the thread ceiling is thread/scheduler collapse.",
round_item: "thread-budget watch (#2419/#2313)",
},
MetricDoc {
name: catalog::PROC_FDS,
kind: MetricKind::Gauge,
unit: catalog::unit::FDS,
summary: "Process open fd count sampled from /proc/self/fd (Linux; absent off-/proc, never 0).",
attributes: &[],
interpretation: "Rises as concurrent scans open SSTables (no reader pool, #815); a level nearing the container ulimit is the fd-exhaustion binding point.",
round_item: "fd-exhaustion watch (#2419/#2313)",
},
MetricDoc {
name: catalog::PROC_RSS_BYTES,
kind: MetricKind::Gauge,
unit: catalog::unit::BYTES,
summary: "Process resident set size from /proc/self/status VmRSS (Linux; absent off-/proc, never 0).",
attributes: &[],
interpretation: "Rises with concurrent scan payloads (Flight bypasses the result-byte budget); a level nearing the memory limit is the OOMKill risk.",
round_item: "memory-pressure watch (#2419/#2313)",
},
MetricDoc {
name: catalog::FLIGHT_BLOCKING_TASKS_IN_USE,
kind: MetricKind::Gauge,
unit: catalog::unit::THREADS,
summary: "Flight spawn_blocking tasks currently outstanding (flight-managed proxy, NOT the global tokio pool queue depth).",
attributes: &[],
interpretation: "Rises with concurrent do_get scans and returns to baseline; a level pinned near the blocking-pool size with flat rpc.rows is blocking-pool saturation. Distinct from admission.in_use.",
round_item: "blocking-pool pressure watch (#2419/#2313)",
},
MetricDoc {
name: catalog::FLIGHT_TABLES_DISCOVERED,
kind: MetricKind::Gauge,
unit: catalog::unit::ENTRIES,
summary: "Table dirs visible under --data-dir, re-sampled by a readdir-only walk on the ~2s saturation tick (no SSTable opens). Undersampled at ~2s vs a longer scrape interval (#2661).",
attributes: &[],
interpretation: "Rises when a table appears on disk and falls when one is removed; a wrong or empty --data-dir reads 0 immediately (an inert mount, visible before the first query errors).",
round_item: "inert-mount watch (#2684)",
},
MetricDoc {
name: catalog::FLIGHT_WARM_TABLES,
kind: MetricKind::Gauge,
unit: catalog::unit::ENTRIES,
summary: "Tables with a live warm reader set in the flight WarmTableRegistry (atomic-backed at the registry mutation sites, not sampler-driven).",
attributes: &[],
interpretation: "Rises on the first serve of a previously-unseen table and falls on eviction/retirement; pinned at capacity with steady eviction churn means the warm byte budget is small vs the working set. Distinct from tables_discovered (visible on disk).",
round_item: "warm-working-set watch (#2684)",
},
];