1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
# yaml-language-server: $schema=https://raw.githubusercontent.com/panchenkoai/rivet/main/schemas/latest/rivet.schema.json
#
# Rivet Example: PostgreSQL → Full Export → Azure Blob Storage (SAS token)
#
# Use case: daily snapshot of production tables to Azure Blob using a
# short-lived SAS token instead of the full account key.
# Least-privilege: the token is scoped to one container and
# expires after a fixed window.
#
# Prerequisites:
# 1. Create the container in the Azure portal (Rivet does not auto-create it).
# 2. Generate a SAS token (portal → Storage account → Shared access signature,
# or via the Azure CLI):
#
# az storage container generate-sas \
# --account-name <account> \
# --name <container> \
# --permissions rwdl \
# --expiry $(date -u -v+7d +%Y-%m-%dT%H:%MZ) \
# --account-key "$AZURE_ACCOUNT_KEY" \
# -o tsv
#
# 3. Export credentials:
# export DATABASE_URL="postgresql://rivet:rivet@localhost:5432/rivet"
# export AZURE_STORAGE_SAS_TOKEN="sv=2021-08-06&ss=b&srt=o&sp=rwdl&..."
#
# Run:
# rivet doctor -c examples/pg_full_azure_sas.yaml
# rivet run -c examples/pg_full_azure_sas.yaml
# rivet validate -c examples/pg_full_azure_sas.yaml
source:
type: postgres
url_env: DATABASE_URL
environment: production
tuning:
profile: balanced
batch_size: 10000
throttle_ms: 50
exports:
- name: users
query: >
SELECT id, name, email, age, balance, is_active, bio, created_at, updated_at
FROM users
mode: full
format: parquet
compression: zstd
columns:
balance: decimal(12,2)
meta_columns:
exported_at: true
destination:
type: azure
bucket: test-container
account_name: rivet
sas_token_env: AZURE_STORAGE_SAS_TOKEN
prefix: "pg/{date}/"
- name: orders
query: >
SELECT id, user_id, product, quantity, price, status, notes, ordered_at, updated_at
FROM orders
mode: full
format: parquet
compression: zstd
columns:
price: decimal(10,2)
meta_columns:
exported_at: true
destination:
type: azure
bucket: test-container
account_name: rivet
sas_token_env: AZURE_STORAGE_SAS_TOKEN
prefix: "pg/{date}/"
- name: events
query: >
SELECT id, user_id, event_type, ip_address, created_at
FROM events
mode: full
format: parquet
compression: zstd
meta_columns:
exported_at: true
destination:
type: azure
bucket: test-container
account_name: rivet
sas_token_env: AZURE_STORAGE_SAS_TOKEN
prefix: "pg/{date}/"