rivet-cli 0.2.0-beta.2

CLI tool to export PostgreSQL and MySQL to Parquet/CSV (local, S3, GCS) with tuning, preflight checks, and SQLite-backed state.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43

# Chunked exports to real GCS bucket rivet_data_test.
# Uses tables from dev/postgres/init.sql (docker-compose).

source:
  type: postgres
  url: "postgresql://rivet:rivet@localhost:5432/rivet"
  tuning:
    profile: fast
  batch_size: 1000

exports:
  - name: content_items_gcs
    query: "SELECT * FROM content_items"
    mode: chunked
    chunk_column: id
    chunk_size: 100000
    format: parquet
    compression: zstd
    destination:
      type: gcs
      bucket: rivet_data_test
      prefix: exports/orders/
      credentials_file: /Users/andriipanchenko/rivet/rivet-gcs-sa.json

  - name: page_views_chunked_gcs
    query: >
      SELECT id, session_id, user_id, url, referrer, user_agent, ip_address,
             country_code, region, city, device_type, browser, os,
             screen_width, screen_height, viewport_width, viewport_height,
             page_load_ms, dom_ready_ms, time_on_page_ms, scroll_depth_pct,
             click_count, is_bounce, utm_source, utm_medium, utm_campaign,
             utm_term, utm_content, custom_props::text as custom_props, created_at
      FROM page_views
    mode: chunked
    chunk_column: id
    chunk_size: 400000
    parallel: 5
    format: parquet
    destination:
      type: gcs
      bucket: rivet_data_test
      prefix: exports/page_views/
      credentials_file: /Users/andriipanchenko/rivet/rivet-gcs-sa.json