rivet-cli 0.16.3

Rivet: PostgreSQL/MySQL/SQL Server → Parquet/CSV (local, S3, GCS, Azure). Crate name rivet-cli; binary rivet.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32

# Rivet MySQL benchmark config — what we ran to produce REPORT_mysql.md.
#
# Same `table:` + chunked + `chunk_size_memory_mb` strategy as the PG config.
# `chunk_size_memory_mb: 1024` on the bigger MySQL chunks because MySQL's
# `AVG_ROW_LENGTH` over-counts BLOB-overflow pages (corrected by /3 in the
# introspection helper) and the client-side `row_buf` is independently capped
# at ~64 MB Arrow via the size-cap in `mysql_run_export`. Net result: ~26 files
# on content_items with peak RSS ~270 MB.

source:
  type: mysql
  url: "mysql://rivet:rivet@127.0.0.1:3306/rivet"
  environment: local

exports:
  - { name: audit_log,              table: audit_log,              mode: chunked, chunk_size_memory_mb: 1024, format: parquet, compression: snappy, destination: { type: local, path: /tmp/rivet_bench/output_mysql/rivet/audit_log } }
  - { name: content_items,          table: content_items,          mode: chunked, chunk_size_memory_mb: 1024, format: parquet, compression: snappy, destination: { type: local, path: /tmp/rivet_bench/output_mysql/rivet/content_items } }
  - { name: email_queue,            table: email_queue,            mode: chunked, chunk_size_memory_mb: 1024, format: parquet, compression: snappy, destination: { type: local, path: /tmp/rivet_bench/output_mysql/rivet/email_queue } }
  - { name: events,                 table: events,                 mode: chunked, chunk_size_memory_mb: 256,  format: parquet, compression: snappy, destination: { type: local, path: /tmp/rivet_bench/output_mysql/rivet/events } }
  - { name: logs_archive,           table: logs_archive,           mode: chunked, chunk_size_memory_mb: 1024, format: parquet, compression: snappy, destination: { type: local, path: /tmp/rivet_bench/output_mysql/rivet/logs_archive } }
  - { name: metric_samples,         table: metric_samples,         mode: chunked, chunk_size_memory_mb: 1024, format: parquet, compression: snappy, destination: { type: local, path: /tmp/rivet_bench/output_mysql/rivet/metric_samples } }
  - { name: orders,                 table: orders,                 mode: chunked, chunk_size_memory_mb: 256,  format: parquet, compression: snappy, destination: { type: local, path: /tmp/rivet_bench/output_mysql/rivet/orders }, columns: { price: "decimal(10,2)" } }
  - { name: orders_coalesce,        table: orders_coalesce,        mode: chunked, chunk_size_memory_mb: 256,  format: parquet, compression: snappy, destination: { type: local, path: /tmp/rivet_bench/output_mysql/rivet/orders_coalesce }, columns: { price: "decimal(10,2)" } }
  - { name: orders_sparse,          table: orders_sparse,          mode: full,                                  format: parquet, compression: snappy, destination: { type: local, path: /tmp/rivet_bench/output_mysql/rivet/orders_sparse } }
  - { name: orders_sparse_for_export, table: orders_sparse_for_export, mode: full,                              format: parquet, compression: snappy, destination: { type: local, path: /tmp/rivet_bench/output_mysql/rivet/orders_sparse_for_export } }
  - { name: page_views,             table: page_views,             mode: chunked, chunk_size_memory_mb: 1024, format: parquet, compression: snappy, destination: { type: local, path: /tmp/rivet_bench/output_mysql/rivet/page_views } }
  - { name: product_catalog,        table: product_catalog,        mode: full,                                  format: parquet, compression: snappy, destination: { type: local, path: /tmp/rivet_bench/output_mysql/rivet/product_catalog }, columns: { price: "decimal(10,2)" } }
  - { name: rivet_type_matrix,      table: rivet_type_matrix,      mode: full,                                  format: parquet, compression: snappy, destination: { type: local, path: /tmp/rivet_bench/output_mysql/rivet/rivet_type_matrix }, columns: { amount: "decimal(18,2)", fee: "decimal(18,6)" } }
  - { name: rivet_type_matrix_full, table: rivet_type_matrix_full, mode: full,                                  format: parquet, compression: snappy, destination: { type: local, path: /tmp/rivet_bench/output_mysql/rivet/rivet_type_matrix_full } }
  - { name: sessions,               table: sessions,               mode: chunked, chunk_size_memory_mb: 256,  format: parquet, compression: snappy, destination: { type: local, path: /tmp/rivet_bench/output_mysql/rivet/sessions } }
  - { name: transactions,           table: transactions,           mode: chunked, chunk_size_memory_mb: 1024, format: parquet, compression: snappy, destination: { type: local, path: /tmp/rivet_bench/output_mysql/rivet/transactions }, columns: { amount: "decimal(12,2)" } }
  - { name: users,                  table: users,                  mode: full,                                  format: parquet, compression: snappy, destination: { type: local, path: /tmp/rivet_bench/output_mysql/rivet/users }, columns: { balance: "decimal(12,2)" } }