Skip to main content

rete_core/
write_opts.rs

1//! Writer tuning knobs: index tile size, dictionary chunk size, zstd level and
2//! frame shape, and frequency-promoted object ids. The measurements behind the
3//! defaults are in `docs/BENCHMARK.md` ("Compression (0.4)").
4//!
5//! None of these change the file format. Every combination writes a file any
6//! reader of the same format version opens unchanged: tiles and dictionary
7//! chunks of any size are routed by their directories, zstd frames of any level
8//! or shape decode with any conformant decoder, and a promoted object is an
9//! ordinary entry of the shared dictionary section.
10//!
11//! The settings are **process-wide**, like `RETE_DICT_RESTART_INTERVAL`: the
12//! CLI sets them once, before a build, and every writer path (the in-memory and
13//! streaming assemblers and the external build) reads the same values, so the
14//! byte-identity between those paths holds for every setting. Each knob falls
15//! back to an environment variable, then to the built-in default:
16//!
17//! | knob | env var | default |
18//! |---|---|---|
19//! | index tile budget | `RETE_TILE_KIB` | 64 KiB |
20//! | dictionary chunk budget | `RETE_DICT_CHUNK_KIB` | 64 KiB |
21//! | zstd level | `RETE_ZSTD_LEVEL` | 9 |
22//! | zstd frame shape | `RETE_ZSTD_FRAME` (`sized` / `stream`) | `sized` |
23//! | frequency-promoted objects | `RETE_FREQUENT_OBJECTS` | 0 (off) |
24
25use std::sync::atomic::{AtomicI32, AtomicU8, AtomicUsize, Ordering};
26
27/// Default per-tile budget of an index permutation, in uncompressed bytes.
28pub const DEFAULT_TILE_BUDGET_BYTES: usize = 64 * 1024;
29/// Default per-chunk budget of a dictionary section body, in uncompressed bytes.
30pub const DEFAULT_DICT_CHUNK_BYTES: usize = 64 * 1024;
31/// Default zstd level of the writer.
32pub const DEFAULT_ZSTD_LEVEL: i32 = 9;
33/// Smallest accepted tile or chunk budget (KiB). Below this the per-unit
34/// directory entry and frame header dominate.
35pub const MIN_BUDGET_KIB: usize = 4;
36/// Largest accepted tile or chunk budget (KiB). A tile is decoded whole on
37/// every touch, so very large tiles turn a point lookup into a large decode.
38pub const MAX_BUDGET_KIB: usize = 4096;
39
40/// How each tile / chunk is framed by zstd.
41#[derive(Debug, Clone, Copy, PartialEq, Eq)]
42pub enum FrameShape {
43    /// A streaming frame without a content size: what every writer up to 0.3
44    /// produced, and still available (`RETE_ZSTD_FRAME=stream`) to reproduce a
45    /// 0.3 file byte for byte. Not knowing the input size, zstd compresses each
46    /// 64 KiB unit with the parameters it tunes for large inputs, and declares
47    /// the level's whole window (4 MiB at level 9) in the frame header.
48    Stream,
49    /// A single-segment frame that records its content size (the default since
50    /// 0.4). zstd then picks the parameter set it tunes for inputs of this size:
51    /// at level 9 the index comes out 8% smaller on the OpenCitations sample,
52    /// and encoding is faster; the frame declares the tile's own size instead
53    /// of a 4 MiB window. Any conformant decoder reads both shapes, so this is
54    /// not a format change (released 0.3.5 and 0.3.7 readers verified).
55    Sized,
56}
57
58/// A complete set of writer settings.
59#[derive(Debug, Clone, Copy, PartialEq, Eq)]
60pub struct WriteOptions {
61    /// Index tile budget in uncompressed bytes.
62    pub tile_budget: usize,
63    /// Dictionary chunk budget in uncompressed bytes.
64    pub dict_chunk_budget: usize,
65    /// zstd compression level (1..=22).
66    pub zstd_level: i32,
67    /// zstd frame shape.
68    pub frame: FrameShape,
69    /// How many of the most frequent object-only terms to promote into the
70    /// shared dictionary section (0 = off).
71    pub frequent_objects: usize,
72}
73
74impl Default for WriteOptions {
75    fn default() -> Self {
76        WriteOptions {
77            tile_budget: DEFAULT_TILE_BUDGET_BYTES,
78            dict_chunk_budget: DEFAULT_DICT_CHUNK_BYTES,
79            zstd_level: DEFAULT_ZSTD_LEVEL,
80            frame: FrameShape::Sized,
81            frequent_objects: 0,
82        }
83    }
84}
85
86// 0 / -1 / u8::MAX = "not set yet": resolve from the environment on first read.
87static TILE_BUDGET: AtomicUsize = AtomicUsize::new(0);
88static DICT_CHUNK_BUDGET: AtomicUsize = AtomicUsize::new(0);
89static ZSTD_LEVEL: AtomicI32 = AtomicI32::new(0);
90static FRAME: AtomicU8 = AtomicU8::new(u8::MAX);
91static FREQUENT_OBJECTS: AtomicUsize = AtomicUsize::new(usize::MAX);
92
93fn env_parse<T: std::str::FromStr>(name: &str) -> Option<T> {
94    std::env::var(name).ok().and_then(|v| v.trim().parse().ok())
95}
96
97fn kib_env(name: &str, default: usize) -> usize {
98    env_parse::<usize>(name)
99        .filter(|k| (MIN_BUDGET_KIB..=MAX_BUDGET_KIB).contains(k))
100        .map(|k| k * 1024)
101        .unwrap_or(default)
102}
103
104/// The index tile budget in force (bytes).
105pub fn tile_budget() -> usize {
106    match TILE_BUDGET.load(Ordering::Relaxed) {
107        0 => {
108            let v = kib_env("RETE_TILE_KIB", DEFAULT_TILE_BUDGET_BYTES);
109            TILE_BUDGET.store(v, Ordering::Relaxed);
110            v
111        }
112        v => v,
113    }
114}
115
116/// The dictionary chunk budget in force (bytes).
117pub fn dict_chunk_budget() -> usize {
118    match DICT_CHUNK_BUDGET.load(Ordering::Relaxed) {
119        0 => {
120            let v = kib_env("RETE_DICT_CHUNK_KIB", DEFAULT_DICT_CHUNK_BYTES);
121            DICT_CHUNK_BUDGET.store(v, Ordering::Relaxed);
122            v
123        }
124        v => v,
125    }
126}
127
128/// The zstd level in force.
129pub fn zstd_level() -> i32 {
130    match ZSTD_LEVEL.load(Ordering::Relaxed) {
131        0 => {
132            let v = env_parse::<i32>("RETE_ZSTD_LEVEL")
133                .filter(|l| (1..=22).contains(l))
134                .unwrap_or(DEFAULT_ZSTD_LEVEL);
135            ZSTD_LEVEL.store(v, Ordering::Relaxed);
136            v
137        }
138        v => v,
139    }
140}
141
142/// The zstd frame shape in force.
143pub fn frame_shape() -> FrameShape {
144    let code = match FRAME.load(Ordering::Relaxed) {
145        u8::MAX => {
146            let v = match std::env::var("RETE_ZSTD_FRAME").ok().as_deref() {
147                Some("stream") => 0,
148                _ => 1,
149            };
150            FRAME.store(v, Ordering::Relaxed);
151            v
152        }
153        v => v,
154    };
155    if code == 1 {
156        FrameShape::Sized
157    } else {
158        FrameShape::Stream
159    }
160}
161
162#[cfg(test)]
163thread_local! {
164    /// [`frequent_objects`] for this thread alone: the crate's unit tests run
165    /// in parallel threads, so one that promotes must not change what every
166    /// other test's builder does.
167    pub(crate) static TEST_FREQUENT_OBJECTS: std::cell::Cell<Option<usize>> =
168        const { std::cell::Cell::new(None) };
169}
170
171/// How many frequent object-only terms the dictionary builder promotes.
172pub fn frequent_objects() -> usize {
173    #[cfg(test)]
174    if let Some(n) = TEST_FREQUENT_OBJECTS.with(|c| c.get()) {
175        return n;
176    }
177    match FREQUENT_OBJECTS.load(Ordering::Relaxed) {
178        usize::MAX => {
179            let v = env_parse::<usize>("RETE_FREQUENT_OBJECTS").unwrap_or(0);
180            FREQUENT_OBJECTS.store(v, Ordering::Relaxed);
181            v
182        }
183        v => v,
184    }
185}
186
187/// The full set of settings in force.
188pub fn write_options() -> WriteOptions {
189    WriteOptions {
190        tile_budget: tile_budget(),
191        dict_chunk_budget: dict_chunk_budget(),
192        zstd_level: zstd_level(),
193        frame: frame_shape(),
194        frequent_objects: frequent_objects(),
195    }
196}
197
198/// Replace the settings in force for this process (the CLI calls this once,
199/// from its flags, before a build). Values outside the accepted ranges are
200/// clamped.
201pub fn set_write_options(o: WriteOptions) {
202    let clamp_budget = |b: usize| b.clamp(MIN_BUDGET_KIB * 1024, MAX_BUDGET_KIB * 1024);
203    TILE_BUDGET.store(clamp_budget(o.tile_budget), Ordering::Relaxed);
204    DICT_CHUNK_BUDGET.store(clamp_budget(o.dict_chunk_budget), Ordering::Relaxed);
205    ZSTD_LEVEL.store(o.zstd_level.clamp(1, 22), Ordering::Relaxed);
206    FRAME.store(
207        match o.frame {
208            FrameShape::Stream => 0,
209            FrameShape::Sized => 1,
210        },
211        Ordering::Relaxed,
212    );
213    FREQUENT_OBJECTS.store(o.frequent_objects, Ordering::Relaxed);
214}
215
216/// Default rows of one sort run of the `0x07` quad orders: 100 M rows are 1.6 GB
217/// of `(u32, u32, u32, u32)`, which with the tile buffers stays under a 4 GB
218/// container. A file with at most this many named quads is sorted in RAM.
219pub const DEFAULT_QUAD_RUN_ROWS: usize = 100_000_000;
220
221static QUAD_RUN_ROWS: AtomicUsize = AtomicUsize::new(0);
222
223/// Rows per sort run of the `0x07` quad orders (`RETE_QUAD_RUN_ROWS`, default
224/// [`DEFAULT_QUAD_RUN_ROWS`]). Changes how a build uses RAM and disk, never the
225/// bytes it writes.
226pub fn quad_run_rows() -> usize {
227    match QUAD_RUN_ROWS.load(Ordering::Relaxed) {
228        0 => {
229            let v = env_parse::<usize>("RETE_QUAD_RUN_ROWS")
230                .filter(|&n| n > 0)
231                .unwrap_or(DEFAULT_QUAD_RUN_ROWS);
232            QUAD_RUN_ROWS.store(v, Ordering::Relaxed);
233            v
234        }
235        v => v,
236    }
237}
238
239/// Set the rows per sort run (0 = back to the default).
240pub fn set_quad_run_rows(n: usize) {
241    QUAD_RUN_ROWS.store(n, Ordering::Relaxed);
242}
243
244/// Default size above which a `0x07` graph view does not build its POS and OSP
245/// orders from its own GSPO slice (see `quad::view`).
246pub const DEFAULT_QUAD_SYNTH_MAX: u64 = 1 << 20;
247
248static QUAD_SYNTH_MAX: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
249
250/// Quads of a named graph up to which a `0x07` view builds its POS and OSP
251/// orders from the graph's GSPO tiles (`RETE_QUAD_SYNTH_MAX`, default
252/// [`DEFAULT_QUAD_SYNTH_MAX`]); a bigger graph reads them from the graph-last
253/// POSG and OSPG tiles. Changes what a query reads, never what it returns.
254pub fn quad_view_synth_max() -> u64 {
255    match QUAD_SYNTH_MAX.load(Ordering::Relaxed) {
256        0 => {
257            let v = env_parse::<u64>("RETE_QUAD_SYNTH_MAX")
258                .filter(|&n| n > 0)
259                .unwrap_or(DEFAULT_QUAD_SYNTH_MAX);
260            QUAD_SYNTH_MAX.store(v, Ordering::Relaxed);
261            v
262        }
263        v => v,
264    }
265}
266
267/// Set [`quad_view_synth_max`] (0 = back to the default).
268pub fn set_quad_view_synth_max(n: u64) {
269    QUAD_SYNTH_MAX.store(n, Ordering::Relaxed);
270}
271
272/// Class clustering of the subject-leading permutations (`0x07`,
273/// `--cluster-subjects-by-class`).
274#[derive(Debug, Clone, Copy, PartialEq, Eq)]
275pub struct ClusterOpts {
276    /// The most classes that get a run of their own (at most
277    /// [`crate::v06::MAX_CLUSTERS`]).
278    pub max_classes: usize,
279    /// A class needs at least this many subjects to get a run: a smaller class
280    /// would not fill a tile, and every run costs a point lookup one more tile.
281    pub min_subjects: u64,
282    /// The most classes the quad index's SPOG gets a run for. Fewer than the
283    /// default graph's: a bound subject of SPOG reads one leaf in every run, and
284    /// a named-graph dump is far larger than a default graph, so each run costs
285    /// more to look past (SPEC.md section 6.6).
286    pub quad_classes: usize,
287}
288
289/// Default of [`ClusterOpts::min_subjects`].
290pub const DEFAULT_CLUSTER_MIN_SUBJECTS: u64 = 1000;
291
292/// Default of [`ClusterOpts::quad_classes`].
293pub const DEFAULT_CLUSTER_QUAD_CLASSES: usize = 10;
294
295impl Default for ClusterOpts {
296    fn default() -> Self {
297        ClusterOpts {
298            max_classes: crate::v06::MAX_CLUSTERS,
299            min_subjects: DEFAULT_CLUSTER_MIN_SUBJECTS,
300            quad_classes: std::env::var("RETE_CLUSTER_QUAD_CLASSES")
301                .ok()
302                .and_then(|v| v.parse().ok())
303                .unwrap_or(DEFAULT_CLUSTER_QUAD_CLASSES),
304        }
305    }
306}
307
308static CLUSTER: std::sync::Mutex<Option<ClusterOpts>> = std::sync::Mutex::new(None);
309
310/// The clustering the writers apply, `None` = off (the default). Process-wide
311/// like the other writer settings, so every writer path agrees.
312pub fn cluster() -> Option<ClusterOpts> {
313    CLUSTER.lock().ok().and_then(|g| *g)
314}
315
316/// Turn class clustering on (with these limits) or off.
317pub fn set_cluster(c: Option<ClusterOpts>) {
318    if let Ok(mut g) = CLUSTER.lock() {
319        *g = c;
320    }
321}