1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
# MeiliBridge Configuration Example
# Copy this file to config.yaml and update with your settings
# Application metadata
app:
name: "MeiliBridge" # Application name
instance_id: "dev-01" # Unique instance identifier
tags: # Custom tags for identification
environment: "development"
region: "us-east-1"
# Option 1: Single source configuration
# source:
# type: postgresql
# host: "localhost"
# port: 5432
# database: "myapp"
# username: "postgres" # Note: 'username' not 'user'
# password: "${POSTGRES_PASSWORD}"
# pool:
# max_size: 10
# min_idle: 1 # Default is 1, not 2
# connection_timeout: 30
# idle_timeout: 600
# slot_name: "meilibridge_slot"
# publication: "meilibridge_pub"
# ssl:
# mode: "disable" # disable, prefer, require, verify-ca, verify-full
# ca_cert: "/path/to/ca.crt"
# client_cert: "/path/to/client.crt"
# client_key: "/path/to/client.key"
# statement_cache:
# enabled: true
# max_size: 100
# Option 2: Multiple sources configuration (recommended for multi-database)
sources:
- name: "primary"
type: postgresql
host: "localhost"
port: 5432
database: "myapp"
username: "postgres" # Note: 'username' not 'user'
password: "${POSTGRES_PASSWORD}"
pool:
max_size: 10
min_idle: 1 # Default is 1
slot_name: "meilibridge_slot_primary"
publication: "meilibridge_pub_primary"
statement_cache:
enabled: true
max_size: 100
- name: "secondary"
type: postgresql
host: "secondary.example.com"
port: 5432
database: "analytics"
username: "postgres" # Note: 'username' not 'user'
password: "${SECONDARY_DB_PASSWORD}"
pool:
max_size: 5
min_idle: 1
slot_name: "meilibridge_slot_secondary"
publication: "meilibridge_pub_secondary"
statement_cache:
enabled: true
max_size: 50
meilisearch:
url: "http://localhost:7700"
api_key: "${MEILI_MASTER_KEY}"
timeout: 30
max_connections: 10 # Connection pool size
batch_size: 1000 # Default batch size for operations
primary_key: "id" # Default primary key for all indexes
auto_create_index: true # Auto-create missing indexes
# Index settings template (optional)
index_settings:
searchable_attributes: # Fields to search
displayed_attributes: # Fields to return in results
filterable_attributes: # Fields available for filtering
sortable_attributes: # Fields available for sorting
ranking_rules: # Custom ranking rules
stop_words: # Stop words list
synonyms: # Synonyms mapping
# Circuit breaker configuration
circuit_breaker:
enabled: true # Enable circuit breaker protection
error_rate: 0.5 # Open circuit at 50% error rate
min_request_count: 10 # Minimum requests before evaluation
consecutive_failures: 5 # Or 5 consecutive failures
timeout_secs: 60 # Time before attempting recovery
redis:
url: "redis://localhost:6379"
password: "${REDIS_PASSWORD}" # Optional
database: 0 # Redis database number
key_prefix: "meilibridge" # Prefix for all keys
# Connection pool settings
pool:
max_size: 10 # Maximum connections
min_idle: 2 # Minimum idle connections
connection_timeout: 5 # Connection timeout (seconds)
sync_tasks:
# Sync from primary source
- id: "users_sync"
source_name: "primary" # Specify which source to use
table: "public.users"
index: "users"
primary_key: "id"
full_sync_on_start: true
options:
batch_size: 1000
batch_timeout_ms: 1000
# Sync from secondary source
- id: "analytics_events_sync"
source_name: "secondary" # Different source
table: "public.analytics_events"
index: "analytics_events"
primary_key: "event_id"
full_sync_on_start: false # CDC only
options:
batch_size: 500
batch_timeout_ms: 2000
# Soft delete configuration (optional)
soft_delete:
field: "status" # Field to check for soft delete
delete_values: # Values that indicate deletion
- "DELETED"
- "INACTIVE"
- "false" # Can also check boolean fields
handle_on_full_sync: true # Filter out during full sync
handle_on_cdc: true # Transform UPDATE to DELETE during CDC
api:
enabled: true
host: "0.0.0.0"
port: 7701
logging:
level: "info"
format: "pretty"
features:
auto_recovery: true
health_checks: true
metrics_export: true
distributed_mode: false
# Performance configuration
performance:
# Parallel processing for high throughput
parallel_processing:
enabled: false # Enable for high-volume deployments
workers_per_table: 4 # Number of parallel workers per table
max_concurrent_events: 1000 # Max events processed concurrently
work_stealing: true # Enable load balancing between tables
work_steal_interval_ms: 100 # How often to check for work stealing
work_steal_threshold: 50 # Min queue size difference for stealing
# Batch processing settings
batch_processing:
default_batch_size: 100
max_batch_size: 1000
min_batch_size: 10
batch_timeout_ms: 5000
adaptive_batching: true # Enable dynamic batch sizing
# Adaptive batching configuration
adaptive_config:
target_latency_ms: 1000 # Target processing time per batch
adjustment_factor: 0.2 # How aggressively to adjust (0.0-1.0)
metric_window_size: 10 # Number of metrics to average
adjustment_interval_ms: 5000 # Min time between adjustments
memory_pressure_threshold: 80.0 # Memory usage % to reduce batch size
per_table_optimization: true # Optimize batch size per table
# Connection pool settings
connection_pool:
max_connections: 20
min_connections: 5
connection_timeout: 30
idle_timeout: 600
# Error handling configuration
error_handling:
retry:
enabled: true
max_attempts: 3
initial_backoff_ms: 100
max_backoff_ms: 30000
backoff_multiplier: 2.0
jitter_factor: 0.1
dead_letter_queue:
enabled: true
storage: "memory" # Options: memory, redis
max_entries_per_task: 10000
retention_hours: 24
# Monitoring configuration
monitoring:
metrics_enabled: true
metrics_interval_seconds: 60
health_checks_enabled: true
health_check_interval_seconds: 30
# At-least-once delivery configuration with deduplication
at_least_once_delivery:
enabled: true # Enable at-least-once delivery with deduplication
deduplication_window: 10000 # Number of events to track for deduplication
transaction_timeout_secs: 30 # Timeout for two-phase commit transactions
two_phase_commit: true # Use two-phase commit protocol
checkpoint_before_write: true # Save checkpoint atomically before Meilisearch write
# Plugin configuration (optional)
plugins:
directory: "./plugins" # Plugin directory path
enabled: # List of enabled plugins