rete_core/write_opts.rs
1//! Writer tuning knobs: index tile size, dictionary chunk size, zstd level and
2//! frame shape, and frequency-promoted object ids. The measurements behind the
3//! defaults are in `docs/BENCHMARK.md` ("Compression (0.4)").
4//!
5//! None of these change the file format. Every combination writes a file any
6//! reader of the same format version opens unchanged: tiles and dictionary
7//! chunks of any size are routed by their directories, zstd frames of any level
8//! or shape decode with any conformant decoder, and a promoted object is an
9//! ordinary entry of the shared dictionary section.
10//!
11//! The settings are **process-wide**, like `RETE_DICT_RESTART_INTERVAL`: the
12//! CLI sets them once, before a build, and every writer path (the in-memory and
13//! streaming assemblers and the external build) reads the same values, so the
14//! byte-identity between those paths holds for every setting. Each knob falls
15//! back to an environment variable, then to the built-in default:
16//!
17//! | knob | env var | default |
18//! |---|---|---|
19//! | index tile budget | `RETE_TILE_KIB` | 64 KiB |
20//! | dictionary chunk budget | `RETE_DICT_CHUNK_KIB` | 64 KiB |
21//! | zstd level | `RETE_ZSTD_LEVEL` | 9 |
22//! | zstd frame shape | `RETE_ZSTD_FRAME` (`sized` / `stream`) | `sized` |
23//! | frequency-promoted objects | `RETE_FREQUENT_OBJECTS` | 0 (off) |
24
25use std::sync::atomic::{AtomicI32, AtomicU8, AtomicUsize, Ordering};
26
27/// Default per-tile budget of an index permutation, in uncompressed bytes.
28pub const DEFAULT_TILE_BUDGET_BYTES: usize = 64 * 1024;
29/// Default per-chunk budget of a dictionary section body, in uncompressed bytes.
30pub const DEFAULT_DICT_CHUNK_BYTES: usize = 64 * 1024;
31/// Default zstd level of the writer.
32pub const DEFAULT_ZSTD_LEVEL: i32 = 9;
33/// Smallest accepted tile or chunk budget (KiB). Below this the per-unit
34/// directory entry and frame header dominate.
35pub const MIN_BUDGET_KIB: usize = 4;
36/// Largest accepted tile or chunk budget (KiB). A tile is decoded whole on
37/// every touch, so very large tiles turn a point lookup into a large decode.
38pub const MAX_BUDGET_KIB: usize = 4096;
39
40/// How each tile / chunk is framed by zstd.
41#[derive(Debug, Clone, Copy, PartialEq, Eq)]
42pub enum FrameShape {
43 /// A streaming frame without a content size: what every writer up to 0.3
44 /// produced, and still available (`RETE_ZSTD_FRAME=stream`) to reproduce a
45 /// 0.3 file byte for byte. Not knowing the input size, zstd compresses each
46 /// 64 KiB unit with the parameters it tunes for large inputs, and declares
47 /// the level's whole window (4 MiB at level 9) in the frame header.
48 Stream,
49 /// A single-segment frame that records its content size (the default since
50 /// 0.4). zstd then picks the parameter set it tunes for inputs of this size:
51 /// at level 9 the index comes out 8% smaller on the OpenCitations sample,
52 /// and encoding is faster; the frame declares the tile's own size instead
53 /// of a 4 MiB window. Any conformant decoder reads both shapes, so this is
54 /// not a format change (released 0.3.5 and 0.3.7 readers verified).
55 Sized,
56}
57
58/// A complete set of writer settings.
59#[derive(Debug, Clone, Copy, PartialEq, Eq)]
60pub struct WriteOptions {
61 /// Index tile budget in uncompressed bytes.
62 pub tile_budget: usize,
63 /// Dictionary chunk budget in uncompressed bytes.
64 pub dict_chunk_budget: usize,
65 /// zstd compression level (1..=22).
66 pub zstd_level: i32,
67 /// zstd frame shape.
68 pub frame: FrameShape,
69 /// How many of the most frequent object-only terms to promote into the
70 /// shared dictionary section (0 = off).
71 pub frequent_objects: usize,
72}
73
74impl Default for WriteOptions {
75 fn default() -> Self {
76 WriteOptions {
77 tile_budget: DEFAULT_TILE_BUDGET_BYTES,
78 dict_chunk_budget: DEFAULT_DICT_CHUNK_BYTES,
79 zstd_level: DEFAULT_ZSTD_LEVEL,
80 frame: FrameShape::Sized,
81 frequent_objects: 0,
82 }
83 }
84}
85
86// 0 / -1 / u8::MAX = "not set yet": resolve from the environment on first read.
87static TILE_BUDGET: AtomicUsize = AtomicUsize::new(0);
88static DICT_CHUNK_BUDGET: AtomicUsize = AtomicUsize::new(0);
89static ZSTD_LEVEL: AtomicI32 = AtomicI32::new(0);
90static FRAME: AtomicU8 = AtomicU8::new(u8::MAX);
91static FREQUENT_OBJECTS: AtomicUsize = AtomicUsize::new(usize::MAX);
92
93fn env_parse<T: std::str::FromStr>(name: &str) -> Option<T> {
94 std::env::var(name).ok().and_then(|v| v.trim().parse().ok())
95}
96
97fn kib_env(name: &str, default: usize) -> usize {
98 env_parse::<usize>(name)
99 .filter(|k| (MIN_BUDGET_KIB..=MAX_BUDGET_KIB).contains(k))
100 .map(|k| k * 1024)
101 .unwrap_or(default)
102}
103
104/// The index tile budget in force (bytes).
105pub fn tile_budget() -> usize {
106 match TILE_BUDGET.load(Ordering::Relaxed) {
107 0 => {
108 let v = kib_env("RETE_TILE_KIB", DEFAULT_TILE_BUDGET_BYTES);
109 TILE_BUDGET.store(v, Ordering::Relaxed);
110 v
111 }
112 v => v,
113 }
114}
115
116/// The dictionary chunk budget in force (bytes).
117pub fn dict_chunk_budget() -> usize {
118 match DICT_CHUNK_BUDGET.load(Ordering::Relaxed) {
119 0 => {
120 let v = kib_env("RETE_DICT_CHUNK_KIB", DEFAULT_DICT_CHUNK_BYTES);
121 DICT_CHUNK_BUDGET.store(v, Ordering::Relaxed);
122 v
123 }
124 v => v,
125 }
126}
127
128/// The zstd level in force.
129pub fn zstd_level() -> i32 {
130 match ZSTD_LEVEL.load(Ordering::Relaxed) {
131 0 => {
132 let v = env_parse::<i32>("RETE_ZSTD_LEVEL")
133 .filter(|l| (1..=22).contains(l))
134 .unwrap_or(DEFAULT_ZSTD_LEVEL);
135 ZSTD_LEVEL.store(v, Ordering::Relaxed);
136 v
137 }
138 v => v,
139 }
140}
141
142/// The zstd frame shape in force.
143pub fn frame_shape() -> FrameShape {
144 let code = match FRAME.load(Ordering::Relaxed) {
145 u8::MAX => {
146 let v = match std::env::var("RETE_ZSTD_FRAME").ok().as_deref() {
147 Some("stream") => 0,
148 _ => 1,
149 };
150 FRAME.store(v, Ordering::Relaxed);
151 v
152 }
153 v => v,
154 };
155 if code == 1 {
156 FrameShape::Sized
157 } else {
158 FrameShape::Stream
159 }
160}
161
162#[cfg(test)]
163thread_local! {
164 /// [`frequent_objects`] for this thread alone: the crate's unit tests run
165 /// in parallel threads, so one that promotes must not change what every
166 /// other test's builder does.
167 pub(crate) static TEST_FREQUENT_OBJECTS: std::cell::Cell<Option<usize>> =
168 const { std::cell::Cell::new(None) };
169}
170
171/// How many frequent object-only terms the dictionary builder promotes.
172pub fn frequent_objects() -> usize {
173 #[cfg(test)]
174 if let Some(n) = TEST_FREQUENT_OBJECTS.with(|c| c.get()) {
175 return n;
176 }
177 match FREQUENT_OBJECTS.load(Ordering::Relaxed) {
178 usize::MAX => {
179 let v = env_parse::<usize>("RETE_FREQUENT_OBJECTS").unwrap_or(0);
180 FREQUENT_OBJECTS.store(v, Ordering::Relaxed);
181 v
182 }
183 v => v,
184 }
185}
186
187/// The full set of settings in force.
188pub fn write_options() -> WriteOptions {
189 WriteOptions {
190 tile_budget: tile_budget(),
191 dict_chunk_budget: dict_chunk_budget(),
192 zstd_level: zstd_level(),
193 frame: frame_shape(),
194 frequent_objects: frequent_objects(),
195 }
196}
197
198/// Replace the settings in force for this process (the CLI calls this once,
199/// from its flags, before a build). Values outside the accepted ranges are
200/// clamped.
201pub fn set_write_options(o: WriteOptions) {
202 let clamp_budget = |b: usize| b.clamp(MIN_BUDGET_KIB * 1024, MAX_BUDGET_KIB * 1024);
203 TILE_BUDGET.store(clamp_budget(o.tile_budget), Ordering::Relaxed);
204 DICT_CHUNK_BUDGET.store(clamp_budget(o.dict_chunk_budget), Ordering::Relaxed);
205 ZSTD_LEVEL.store(o.zstd_level.clamp(1, 22), Ordering::Relaxed);
206 FRAME.store(
207 match o.frame {
208 FrameShape::Stream => 0,
209 FrameShape::Sized => 1,
210 },
211 Ordering::Relaxed,
212 );
213 FREQUENT_OBJECTS.store(o.frequent_objects, Ordering::Relaxed);
214}
215
216/// Default rows of one sort run of the `0x07` quad orders: 100 M rows are 1.6 GB
217/// of `(u32, u32, u32, u32)`, which with the tile buffers stays under a 4 GB
218/// container. A file with at most this many named quads is sorted in RAM.
219pub const DEFAULT_QUAD_RUN_ROWS: usize = 100_000_000;
220
221static QUAD_RUN_ROWS: AtomicUsize = AtomicUsize::new(0);
222
223/// Rows per sort run of the `0x07` quad orders (`RETE_QUAD_RUN_ROWS`, default
224/// [`DEFAULT_QUAD_RUN_ROWS`]). Changes how a build uses RAM and disk, never the
225/// bytes it writes.
226pub fn quad_run_rows() -> usize {
227 match QUAD_RUN_ROWS.load(Ordering::Relaxed) {
228 0 => {
229 let v = env_parse::<usize>("RETE_QUAD_RUN_ROWS")
230 .filter(|&n| n > 0)
231 .unwrap_or(DEFAULT_QUAD_RUN_ROWS);
232 QUAD_RUN_ROWS.store(v, Ordering::Relaxed);
233 v
234 }
235 v => v,
236 }
237}
238
239/// Set the rows per sort run (0 = back to the default).
240pub fn set_quad_run_rows(n: usize) {
241 QUAD_RUN_ROWS.store(n, Ordering::Relaxed);
242}
243
244/// Default size above which a `0x07` graph view does not build its POS and OSP
245/// orders from its own GSPO slice (see `quad::view`).
246pub const DEFAULT_QUAD_SYNTH_MAX: u64 = 1 << 20;
247
248static QUAD_SYNTH_MAX: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
249
250/// Quads of a named graph up to which a `0x07` view builds its POS and OSP
251/// orders from the graph's GSPO tiles (`RETE_QUAD_SYNTH_MAX`, default
252/// [`DEFAULT_QUAD_SYNTH_MAX`]); a bigger graph reads them from the graph-last
253/// POSG and OSPG tiles. Changes what a query reads, never what it returns.
254pub fn quad_view_synth_max() -> u64 {
255 match QUAD_SYNTH_MAX.load(Ordering::Relaxed) {
256 0 => {
257 let v = env_parse::<u64>("RETE_QUAD_SYNTH_MAX")
258 .filter(|&n| n > 0)
259 .unwrap_or(DEFAULT_QUAD_SYNTH_MAX);
260 QUAD_SYNTH_MAX.store(v, Ordering::Relaxed);
261 v
262 }
263 v => v,
264 }
265}
266
267/// Set [`quad_view_synth_max`] (0 = back to the default).
268pub fn set_quad_view_synth_max(n: u64) {
269 QUAD_SYNTH_MAX.store(n, Ordering::Relaxed);
270}
271
272/// Class clustering of the subject-leading permutations (`0x07`,
273/// `--cluster-subjects-by-class`).
274#[derive(Debug, Clone, Copy, PartialEq, Eq)]
275pub struct ClusterOpts {
276 /// The most classes that get a run of their own (at most
277 /// [`crate::v06::MAX_CLUSTERS`]).
278 pub max_classes: usize,
279 /// A class needs at least this many subjects to get a run: a smaller class
280 /// would not fill a tile, and every run costs a point lookup one more tile.
281 pub min_subjects: u64,
282 /// The most classes the quad index's SPOG gets a run for. Fewer than the
283 /// default graph's: a bound subject of SPOG reads one leaf in every run, and
284 /// a named-graph dump is far larger than a default graph, so each run costs
285 /// more to look past (SPEC.md section 6.6).
286 pub quad_classes: usize,
287}
288
289/// Default of [`ClusterOpts::min_subjects`].
290pub const DEFAULT_CLUSTER_MIN_SUBJECTS: u64 = 1000;
291
292/// Default of [`ClusterOpts::quad_classes`].
293pub const DEFAULT_CLUSTER_QUAD_CLASSES: usize = 10;
294
295impl Default for ClusterOpts {
296 fn default() -> Self {
297 ClusterOpts {
298 max_classes: crate::v06::MAX_CLUSTERS,
299 min_subjects: DEFAULT_CLUSTER_MIN_SUBJECTS,
300 quad_classes: std::env::var("RETE_CLUSTER_QUAD_CLASSES")
301 .ok()
302 .and_then(|v| v.parse().ok())
303 .unwrap_or(DEFAULT_CLUSTER_QUAD_CLASSES),
304 }
305 }
306}
307
308static CLUSTER: std::sync::Mutex<Option<ClusterOpts>> = std::sync::Mutex::new(None);
309
310/// The clustering the writers apply, `None` = off (the default). Process-wide
311/// like the other writer settings, so every writer path agrees.
312pub fn cluster() -> Option<ClusterOpts> {
313 CLUSTER.lock().ok().and_then(|g| *g)
314}
315
316/// Turn class clustering on (with these limits) or off.
317pub fn set_cluster(c: Option<ClusterOpts>) {
318 if let Ok(mut g) = CLUSTER.lock() {
319 *g = c;
320 }
321}