1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
use std::time::Duration;
/// Configuration for the engine behaviour
///
/// The defaults are optimal so please only modify these if you know
/// deliberately why you are modifying them.
#[derive(Clone, Copy, Debug)]
pub struct EngineOptions {
/// Interval for refreshing node membership information
pub node_membership_refresh_interval: Duration,
/// Interval for checking node membership status
pub node_membership_check_interval: Duration,
/// Interval for cleaning up inactive nodes from the cluster
pub node_membership_cleanup_interval: Duration,
/// Interval for garbage collecting expired changefeed data
pub changefeed_gc_interval: Duration,
/// Interval for running the index compaction process
///
/// The index compaction thread runs at this interval to process indexes
/// that have been marked for compaction. Compaction helps optimize index
/// performance, particularly for full-text indexes, by consolidating
/// changes and removing unnecessary data.
///
/// Default: 5 seconds
pub index_compaction_interval: Duration,
/// Interval for resuming index builds stranded by a crashed or expired
/// owner node.
///
/// A `CONCURRENTLY` index build runs as a detached task; if its owning node
/// dies mid-build, nothing waits on that generation again, so the durable
/// build state is stuck in `Building`/`Closing` and the index reports
/// `status: indexing` with a frozen counter indefinitely. This task
/// periodically adopts such builds (once their owner lease has expired) and
/// drives them to completion. Set to `Duration::ZERO` to disable and recover
/// stalled builds manually with `REBUILD INDEX`.
///
/// Default: 30 seconds
pub index_build_resume_interval: Duration,
/// Interval for processing queued async events.
///
/// Default: 5 seconds
pub event_processing_interval: Duration,
/// Interval at which the per-node live-query router tails the dedicated
/// `lqe` keyspace and delivers notifications off the write path.
///
/// Only active when the live-query engine is `Router`; under the default
/// `Inline` engine the task is a cheap no-op. This is a poll-based delivery
/// cadence, so it bounds steady-state notification latency — kept short by
/// default. (A commit-driven hot path that removes the poll latency is
/// planned; this interval remains the durable backstop.)
///
/// Default: 100 milliseconds
pub live_query_router_interval: Duration,
/// Interval for the background reclaim of tombstoned namespace/database/
/// index data.
///
/// `REMOVE NAMESPACE/DATABASE/INDEX` delete only the catalog definition and
/// enqueue the data prefix; this task periodically destroys the orphaned
/// data out-of-band.
///
/// Default: 60 seconds
pub reclaim_interval: Duration,
/// Minimum age a tombstoned namespace/database/index must reach before its
/// data is physically reclaimed.
///
/// This is a snapshot-safety grace period, not a convenience delay. The
/// reclaim task destroys data out-of-band — on TiKV via `unsafe_destroy_range`,
/// which bypasses MVCC — so a read transaction whose snapshot predates the
/// `REMOVE` must be given time to finish before its data is physically
/// removed. A removal is only reclaimed once it is older than this window,
/// which is equivalent to requiring its commit timestamp to fall behind the
/// MVCC GC safepoint.
///
/// This MUST be `>= tikv_gc_lifetime` (the safepoint lag): a removal older
/// than the grace is also older than the safepoint, so any transaction that
/// could still read it has already expired. The background-task scheduler
/// enforces this by using `max(reclaim_grace, tikv_gc_lifetime)`, so raising
/// `tikv_gc_lifetime` alone can never make reclaim unsafe.
///
/// Default: 10 minutes (matches the default `tikv_gc_lifetime`)
pub reclaim_grace: Duration,
/// Interval between TiKV MVCC garbage-collection passes.
///
/// Each pass calls `cleanup_locks` followed by `update_safepoint`,
/// allowing TiKV to reclaim space taken by superseded MVCC versions.
/// Mirrors TiDB's default of 10 minutes. Set to `Duration::ZERO` to
/// disable scheduling (the value is also gated by
/// `SURREAL_TIKV_GC_ENABLED`).
///
/// Only the TiKV backend acts on this interval; other backends ignore
/// the task entirely.
///
/// Default: 10 minutes
pub tikv_gc_interval: Duration,
/// How far behind the current TSO a TiKV GC safepoint is allowed to
/// sit. The actual safepoint passed to `gc()` is
/// `current_timestamp - lifetime`.
///
/// Default: 10 minutes
pub tikv_gc_lifetime: Duration,
/// Interval between standalone TiKV lock-cleanup passes.
///
/// Faster cadence than the full GC pass because stale locks block
/// readers immediately, while version GC can wait.
///
/// Default: 60 seconds
pub tikv_lock_cleanup_interval: Duration,
/// Interval for purging expired durable RPC sessions.
///
/// When RPC session persistence is enabled, client-attached sessions are
/// mirrored to the KV store with an absolute expiry. Expired entries are
/// already dropped lazily on load; this task additionally sweeps the
/// session keyspace so entries that are never loaded again do not
/// accumulate. Set to `Duration::ZERO` to disable the sweep.
///
/// Default: 60 seconds
pub rpc_session_gc_interval: Duration,
}
impl Default for EngineOptions {
fn default() -> Self {
Self {
node_membership_refresh_interval: Duration::from_secs(3),
node_membership_check_interval: Duration::from_secs(15),
node_membership_cleanup_interval: Duration::from_secs(300),
changefeed_gc_interval: Duration::from_secs(30),
index_compaction_interval: Duration::from_secs(5),
index_build_resume_interval: Duration::from_secs(30),
event_processing_interval: Duration::from_secs(5),
live_query_router_interval: Duration::from_millis(100),
reclaim_interval: Duration::from_secs(60),
reclaim_grace: Duration::from_secs(600),
tikv_gc_interval: Duration::from_secs(600),
tikv_gc_lifetime: Duration::from_secs(600),
tikv_lock_cleanup_interval: Duration::from_secs(60),
rpc_session_gc_interval: Duration::from_secs(60),
}
}
}
impl EngineOptions {
pub fn with_node_membership_refresh_interval(mut self, interval: Duration) -> Self {
self.node_membership_refresh_interval = interval;
self
}
pub fn with_node_membership_check_interval(mut self, interval: Duration) -> Self {
self.node_membership_check_interval = interval;
self
}
pub fn with_node_membership_cleanup_interval(mut self, interval: Duration) -> Self {
self.node_membership_cleanup_interval = interval;
self
}
pub fn with_changefeed_gc_interval(mut self, interval: Duration) -> Self {
self.changefeed_gc_interval = interval;
self
}
pub fn with_index_compaction_interval(mut self, interval: Duration) -> Self {
self.index_compaction_interval = interval;
self
}
pub fn with_index_build_resume_interval(mut self, interval: Duration) -> Self {
self.index_build_resume_interval = interval;
self
}
pub fn with_event_processing_interval(mut self, interval: Duration) -> Self {
self.event_processing_interval = interval;
self
}
pub fn with_live_query_router_interval(mut self, interval: Duration) -> Self {
self.live_query_router_interval = interval;
self
}
pub fn with_reclaim_interval(mut self, interval: Duration) -> Self {
self.reclaim_interval = interval;
self
}
pub fn with_reclaim_grace(mut self, grace: Duration) -> Self {
self.reclaim_grace = grace;
self
}
pub fn with_tikv_gc_interval(mut self, interval: Duration) -> Self {
self.tikv_gc_interval = interval;
self
}
pub fn with_tikv_gc_lifetime(mut self, lifetime: Duration) -> Self {
self.tikv_gc_lifetime = lifetime;
self
}
pub fn with_tikv_lock_cleanup_interval(mut self, interval: Duration) -> Self {
self.tikv_lock_cleanup_interval = interval;
self
}
pub fn with_rpc_session_gc_interval(mut self, interval: Duration) -> Self {
self.rpc_session_gc_interval = interval;
self
}
}