rivet-cli 0.20.0

# ─────────────────────────────────────────────────────────────────────────────
# Benchmark matrix — the SINGLE SOURCE OF TRUTH for the cross-tool benchmark.
#
# Everything propagates from here:
#   • the seed fixture  → the Rust `seed` tool (src/bin/seed) is invoked with the
#     sizes below, so the fixture is defined ONCE and applied to every DB;
#   • the tool set + versions + steelman tuning → the run harness invokes each
#     tool at its declared best-effort config;
#   • the report (harm matrix: table × tool × database) renders from the results.
#
# Rules (per the systematization plan):
#   • ALWAYS pin `version` + `version_date`; ALWAYS (re)test on the LATEST version
#     (use `latest` to refresh, then bump version + date).
#   • ONE canonical seed fixture (the Rust seed tool), applied to every database.
#   • record the `hardware` every run was measured on.
#   • smokes first (a small subset) → show results → then the full run.
# ─────────────────────────────────────────────────────────────────────────────

meta:
  updated: "2026-07-09"
  hardware:
    machine: "Apple M1 Pro"
    cpu: "10-core (8 performance + 2 efficiency)"
    ram_gb: 16
    os: "macOS 26.2 (arm64)"
  # A run stamps its own hardware/date into the report; this is the current dev box.

# Source databases the matrix runs against (primary CI versions).
databases:
  - { engine: postgres,  version: "16",   scheme: "postgresql://" }
  - { engine: mysql,     version: "8.0",  scheme: "mysql://" }
  - { engine: sqlserver, version: "2022", scheme: "sqlserver://" }
  - { engine: mongodb,   version: "8.0",  scheme: "mongodb://", note: "JSON-blob model; CI matrix 4.4→8.0. Read by rivet / sling / dlt / ingestr only (duckdb/clickhouse/odbc2parquet are SQL-only)." }

# ─── Seed fixture ─────────────────────────────────────────────────────────────
# Generated by the Rust `seed` tool (src/bin/seed, feature `dev-seed`) — a 27-table
# app-like schema (users / orders / events / page_views / content_items + the
# dimension & lookup tables), applied IDENTICALLY to every source DB. The row
# counts live here and propagate as the tool's args (the harness passes them),
# so the fixture size is declared once. Derived sizes:
#   orders ≈ users × orders_per_user   ·   events ≈ users × events_per_user
seed:
  tool: "cargo run --release --bin seed --features dev-seed --"
  profile: fast            # fast (SQL-side generate_series) | realistic (COPY) | insert
  target: both             # postgres | mysql | both   (SQL Server seeded via its own path)
  sizes:                   # → passed as `seed --<knob> <value>`
    users: 100000                 # users table
    orders_per_user: 10           # orders ≈ 1,000,000
    events_per_user: 50           # events ≈ 5,000,000
    page_views: 2000000           # wide table (degraded / memory-pressure scenario)
    content_items: 2000000        # heavy text (worst case for peak RSS)
    coalesce_rows: 2000           # composite-cursor demo (NULL updated_at)
    coalesce_null_ratio: 0.35
  # MongoDB is a document store — the SQL schema does not apply. Its fixture is a
  # set of collections seeded via the mongo driver (extended-JSON documents), sized
  # here so the same doc counts propagate to every mongo bench run.
  mongo:
    collections:
      - { name: content_items, docs: 200000,  shape: "id + heavy-text document (~1 KB)" }
      - { name: events,        docs: 5000000,  shape: "small event document" }
      - { name: users,         docs: 100000,   shape: "profile document" }

# ─── Tools ───────────────────────────────────────────────────────────────────
# `steelman` gives each competitor its FAIREST shot — we run only steelman (no
# strawman default column; no point running the same tool twice).
#
# THE GOAL, applied identically to EVERY tool (not just rivet): the lowest-memory
# config at which the tool still COMPLETES the export. This is the honest bench:
#   • memory caps on competitors LOWER their RSS — they flatter the competitor,
#     they do not handicap it (a capped duckdb reports LESS peak_mb, not more);
#   • no arbitrary throttles (e.g. thread caps) that only slow a tool without
#     serving the min-memory goal;
#   • every axis is reported side by side (matrix 1 throughput + matrix 2 harm),
#     so any config trade-off is visible — rivet is NOT the fastest (duckdb /
#     clickhouse / ingestr beat its rows/s); it wins memory, query-hold, types.
# That transparency is the fairness. Per tool:
#   applied — the exact tuned settings the harness invokes it with;
#   knobs   — the FULL tuning surface the tool exposes (so the fairness is
#             auditable: every memory/chunk/parallelism lever it has is listed);
#   note    — caveats (paywalled knobs, OOM floors, quirks).
# Verified 2026-07-09.
tools:
  - name: rivet
    role: subject
    lang: Rust
    version: "0.18.0"
    version_date: "2026-07-09"
    latest: "rivet --version"
    steelman:
      applied: "mode: full, tuning.profile: fast, compression: zstd — one clean file, cursor-streamed"
      knobs:   "mode (full/chunked/date-chunked), tuning.profile (safe/balanced/fast — default balanced throttles 50ms/batch), chunk_size (default 100k), chunk_size_memory_mb, chunk_count, batch_size, compression, parallel_exports, meta_columns"
      note:    "profile:fast drops the Balanced 50ms/batch throttle — MEASURED +24% rows/s, -13% source CPU, -12s open-txn, same 56MB peak / 0 type-loss; only cost oltp_p99 4.0→5.1x. zstd is free here (same speed as snappy/none, 4-18x smaller). chunked mode = source-safety for huge tables (separate scenario)"

  - name: sling
    role: competitor
    lang: Go
    version: "1.5.20"
    version_date: "2026-07-09"
    latest: "sling --version   # brew install slingdata-io/sling/sling"
    steelman:
      applied: "tgt format parquet, file_max_rows=250000 (rotate parts — required to complete at scale)"
      knobs:   "chunk_size (Pro/paywalled), target file_max_rows / file_max_bytes, SLING_THREADS, source range/limit, sslmode=disable"
      note:    "no native parquet writer — streams source→CSV→local HTTP→duckdb read_csv→parquet, which TIMES OUT on a single 2M-row stream; file_max_rows rotation is what makes it complete"

  - name: ingestr
    role: competitor
    lang: Go
    version: "1.0.73"
    version_date: "2026-07-09"
    latest: "curl -LsSf https://getbruin.com/install/ingestr | sh   # or download the Darwin_arm64 release tarball"
    steelman:
      applied: "default streaming; parquet:// dest, sslmode=disable"
      knobs:   "--loader-file-size, --interval-start/--interval-end (incremental), --sql-limit, --page-size (source cursor)"
      note:    "single-binary Go CLI; streams by construction; parquet:// local dest since 1.0.73"

  - name: dlt
    role: competitor
    lang: Python
    version: "1.28.1"
    version_date: "2026-07-09"
    latest: "pip install -U dlt"
    steelman:
      applied: "DATA_WRITER__BUFFER_MAX_ITEMS=5000; loader_file_format=parquet"
      knobs:   "buffer_max_items, file_max_items, file_max_bytes, DATA_WRITER__*, LOAD__WORKERS, extract/normalize parallelism"
      note:    "~1.27 GB tuned; Python + sqlalchemy overhead dominates CPU. Needs a per-engine SQLAlchemy driver: postgres = psycopg2, mysql = pymysql, mssql = pyodbc — pip install alongside dlt[sql_database]"

  - name: duckdb
    role: competitor
    lang: C++
    version: "1.5.4"
    version_date: "2026-07-09"
    latest: "pip install -U duckdb"
    steelman:
      applied: "SET memory_limit='4GB'; postgres_scanner ATTACH READ_ONLY; COPY … TO parquet (zstd)"
      knobs:   "memory_limit, threads, preserve_insertion_order, temp_directory, max_temp_directory_size, parquet row_group_size"
      note:    "lowest COMPLETING limit; OOMs at 512MB/2GB (its floor, not a choice)"

  - name: clickhouse-local
    role: competitor
    lang: C++
    version: "26.6.1"
    version_date: "2026-07-09"
    latest: "brew install clickhouse   # unsigned cask: xattr -dr com.apple.quarantine <bin>; codesign -s - <bin>"
    steelman:
      applied: "--max_memory_usage=4000000000 (lowest completing floor); threads at default (all cores); postgresql() → INTO OUTFILE FORMAT Parquet"
      knobs:   "max_memory_usage, max_threads, max_block_size, max_bytes_before_external_group_by, output_format_parquet_row_group_size"
      note:    "OOMs (code 241) below its floor; the memory cap lowers its RSS (flatters it). No thread cap — that would only slow it without serving the min-memory goal"

  - name: odbc2parquet
    role: competitor
    lang: Rust
    version: "11.0.1"
    version_date: "2026-07-09"
    latest: "cargo install odbc2parquet"
    steelman:
      applied: "--batch-size-memory 256Mb; one SELECT * per table"
      knobs:   "--batch-size-memory, --batch-size-row, --row-groups-per-file, --file-size-threshold"
      note:    "needs a VENDOR ODBC DRIVER per engine (portability cost): postgres = PostgreSQL Unicode (psqlodbc); mysql = MariaDB Unicode (brew install mariadb-connector-odbc, register the .dylib via odbcinst -i -d), Server=127.0.0.1 to force TCP; mssql = ODBC Driver 18 for SQL Server. No mongo (not SQL)."

# ─── Metric catalog — the SINGLE SOURCE for what the harness reports ──────────
# The run harness (dev/bench/smoke.py) READS this: `key` names the value its
# capture registry produces, `label` is how it prints. A guard fails the run if a
# declared key has no capture (or a captured metric is undeclared) — no drift
# between the yaml and the code. Three matrices come straight from here.

# Matrix 1 — benchmark: throughput + correctness (one row per tool).
benchmark_metrics:
  - { key: status,      label: status, desc: "ok | mismatch | error" }
  - { key: wall_s,      label: wall_s, unit: s,     desc: "total wall time" }
  - { key: rows,        label: rows,   unit: count, desc: "rows exported (correctness gate — must equal source)" }
  - { key: rows_s,      label: rows_s, unit: count, desc: "extraction throughput (rows/wall)" }
  - { key: peak_rss_mb, label: peak_mb, unit: MB,   desc: "peak resident memory of the tool process" }
  - { key: out_mb,      label: out_mb, unit: MB,    desc: "parquet output size (*.parquet only)" }
  - { key: out_files,   label: out_files, unit: count, desc: "parquet files written (1 = single file; many = row-group/partition split — affects downstream small-file overhead)" }
  - { key: type_deg,    label: type_deg, unit: count, desc: "columns whose parquet type family drifts from the source — the matrix-3 headline" }

# Matrix 2 — harm to the source (one row per metric, one column per tool),
# grouped by blast surface. `universal` axes are the same CONCEPT on every engine
# (each implements them via its own DMVs); the per-engine blocks are the native
# counters that differ. smoke.py (postgres) renders universal + postgres.
harm_metrics:
  universal:    # measurable on every engine — the cross-engine harm axes
    - { key: oltp_x,   label: oltp_p99_x, group: "ground truth", desc: "co-running OLTP point-read probe: p99 latency during the extract ÷ idle baseline (the number a DBA signs off)" }
    - { key: cache_mb, label: cache_mb,   group: "cache",        desc: "buffer-pool pages holding the table after the run — the hot working set the extract displaced" }
    - { key: locks,    label: locks_peak, group: "contention",   desc: "peak locks held on the source" }
    - { key: blocked,  label: blocked_peak, group: "contention", desc: "peak OTHER sessions blocked by the tool (locks only harm when they block)" }
    - { key: longq,    label: longq_s,    group: "query hold",   desc: "longest single active query held (source-safety)" }
    - { key: q_p50,    label: q_p50_s,    group: "query hold",   desc: "median active query duration" }
    - { key: longtxn,  label: longtxn_s,  group: "query hold",   desc: "longest open transaction" }
    - { key: conns,    label: conns_peak, group: "query hold",   desc: "peak concurrent connections held" }
    - { key: cpu_ms,   label: cpu_ms,     group: "resource",     desc: "source CPU consumed (cgroup v2 cpu.stat)" }
  postgres:     # pg_stat_database deltas + pg-native gauges
    - { key: xmin_age,      label: xmin_held_xid, group: "mvcc/vacuum", desc: "xid age the tool pins backend_xmin to — how far back it starves VACUUM (DB-wide bloat)" }
    - { key: dead,          label: dead_tup,   group: "mvcc/vacuum", desc: "dead tuples created (≈0 for a pure read)" }
    - { key: tup_returned,  label: tup_read,   group: "i/o",  desc: "tuples read from the source (read amplification, incl. catalog)" }
    - { key: tup_fetched,   label: tup_fetch,  group: "i/o",  desc: "tuples fetched" }
    - { key: temp_mb,       label: temp_mb,    group: "i/o",  desc: "temp-file spill (a naive SELECT * spills; chunked does not)" }
    - { key: blks_read,     label: blks_read,  group: "i/o",  desc: "disk blocks read" }
    - { key: blk_read_time, label: blkread_ms, group: "i/o",  desc: "time spent reading blocks" }
    - { key: wal_kb,        label: wal_kb,     group: "resource", desc: "WAL generated (≈0 for reads; matters for CDC)" }
  mysql:        # SHOW GLOBAL STATUS deltas
    - { key: Innodb_rows_read,                 desc: "rows read from InnoDB" }
    - { key: Innodb_buffer_pool_reads,         desc: "physical page reads (cache miss)" }
    - { key: Innodb_buffer_pool_read_requests, desc: "logical page reads" }
    - { key: Innodb_log_waits,                 desc: "log-buffer waits (write pressure)" }
    - { key: tmp_disk_tables,                  desc: "temp tables spilled to disk" }
    - { key: Handler_read_rnd_next,            desc: "full-scan row reads" }
  sqlserver:    # sys.dm_exec_* / SET STATISTICS IO
    - { key: logical_reads,   desc: "logical page reads" }
    - { key: physical_reads,  desc: "physical page reads" }
    - { key: worker_time_ms,  desc: "CPU worker time on the source" }
  mongodb:      # serverStatus deltas (needs clusterMonitor / serverStatus privilege)
    - { key: mongo_docs_scanned,        desc: "documents examined (scannedObjects) — read amplification" }
    - { key: mongo_keys_scanned,        desc: "index keys examined" }
    - { key: mongo_docs_returned,       desc: "documents returned to the client" }
    - { key: mongo_op_query,            desc: "query opcounter" }
    - { key: mongo_op_getmore,          desc: "getMore opcounter (cursor batches — streaming footprint)" }
    - { key: mongo_wt_cache_bytes_read, desc: "WiredTiger bytes read into cache (I/O vs cache split)" }

# ─── Smoke ───────────────────────────────────────────────────────────────────
# Run first, show results, before the full matrix. A tiny seed (fast), so a full
# round-trip is a couple of minutes, not an hour. Per-database scenarios: mongo is
# collections (not tables) and only rivet/sling/dlt/ingestr read it, so its shape
# diverges from the SQL engines — the harness runs each scenario's own tool subset.
smoke:
  seed_sizes: { users: 2000, page_views: 20000, content_items: 5000 }
  mongo_seed: { content_items: 5000 }            # docs (collections, not tables)
  scenarios:
    - database: postgres
      objects: [content_items, page_views, orders]   # stress + wide + fact
      tools:   [rivet, sling, ingestr, dlt, duckdb, clickhouse-local, odbc2parquet]
    - database: mongodb
      objects: [content_items]                        # JSON-blob path
      tools:   [rivet, sling, ingestr, dlt]           # the mongo-capable subset