semq 1.1.0

Safe Rust bindings to the SEMQ core: deterministic symbolic encoding of float32 vectors.
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
// Copyright (c) 2026 The SEMQ Group Inc.
// Licensed under the PolyForm Noncommercial License 1.0.0. See LICENSE.md for terms.

//! Floor: how much changes without a cause, bound to where it was measured.

use std::fmt;
use std::hash::{Hash, Hasher};
use std::ptr::{self, NonNull};

use semq_sys as sys;

use crate::config::CodecConfig;
use crate::convert::{hex, FreeOnDrop};
use crate::diff::Diff;
use crate::error::{check, new_error, Error, Result};
use crate::gate::GateOptions;
use crate::ids::{ptr_or_null, IdKind};

/// An envelope of observed variation, bound to the context it was measured
/// in: the config, the id kind and the `state_id` of the reference every
/// null diff was taken against.
///
/// A diff is within the floor when it shares at least one row with its
/// reference, removes none, changes at most `changed_rows / total_rows` of
/// the shared rows, its p99 hamming does not exceed `hamming`, and it
/// changes neither `encoder` nor `encoder_revision`. Applying a floor to a
/// diff of another config, id kind or reference is `Incompatible`. No
/// probabilistic coverage is claimed.
///
/// A floor from [`measure_for`](Self::measure_for) with
/// [`GateOptions::per_row`] also records the per-row data the per-row check
/// of [`Diff::evaluate`](crate::Diff::evaluate) needs:
/// [`max_hamming`](Self::max_hamming), the largest hamming of any changed
/// row of any null, and [`distinct_nulls`](Self::distinct_nulls). A floor
/// without them saves to the JSON form SEMQ 1.0 reads.
///
/// Immutable, freed on drop and safe to share between threads. Equality
/// and hashing are those of the nine fields.
pub struct Floor {
    ptr: NonNull<sys::semq_floor_t>,
    config: CodecConfig,
    id_kind: IdKind,
}

// SAFETY: a floor is immutable after construction and the core keeps no
// global or thread-local state, so sharing and moving it is sound.
unsafe impl Send for Floor {}
unsafe impl Sync for Floor {}

impl Drop for Floor {
    fn drop(&mut self) {
        // SAFETY: the handle came from the core and is freed once.
        unsafe { sys::semq_floor_free(self.ptr.as_ptr()) }
    }
}

impl Floor {
    /// A floor from its fields: `nulls >= 1`, `total_rows >= 1`,
    /// `changed_rows <= total_rows` and `hamming <= config.units_per_row()`,
    /// else `InvalidInput`. `reference_id` is the `state_id` of the
    /// reference the floor applies to.
    pub fn new(
        config: &CodecConfig,
        id_kind: IdKind,
        reference_id: [u8; 32],
        nulls: u64,
        changed_rows: u64,
        total_rows: u64,
        hamming: u64,
    ) -> Result<Floor> {
        let mut out: *mut sys::semq_floor_t = ptr::null_mut();
        let mut err = new_error();
        // SAFETY: the config is a validated struct, `reference_id` holds 32
        // readable bytes and every out-pointer is live for the call.
        let status = unsafe {
            sys::semq_floor_create(
                config.as_raw(),
                id_kind.as_raw(),
                reference_id.as_ptr(),
                nulls,
                changed_rows,
                total_rows,
                hamming,
                &mut out,
                &mut err,
            )
        };
        check(status, &err, "floor")?;
        Self::from_raw(out, "floor")
    }

    /// The envelope of one or more null diffs of one reference; every input
    /// is within the result. The diffs must share config, id kind and
    /// reference (`Incompatible`, `field` = the index of the offending
    /// diff), and each must be a valid null: it shares at least one row
    /// with its reference, adds and removes nothing, and leaves `encoder`
    /// and `encoder_revision` unchanged (`InvalidInput`, `field` = index).
    ///
    /// Takes any iterable of diffs: a `Vec<Diff>` or a slice by reference,
    /// `[&d1, &d2]`, or an iterator. The floor records no per-row data; see
    /// [`measure_for`](Self::measure_for).
    pub fn measure<D: AsRef<Diff>>(null_diffs: impl IntoIterator<Item = D>) -> Result<Floor> {
        Self::measure_for(null_diffs, &GateOptions::new())
    }

    /// [`measure`](Self::measure) that also records what the checks in
    /// `checks` need: with [`GateOptions::per_row`],
    /// [`max_hamming`](Self::max_hamming) and
    /// [`distinct_nulls`](Self::distinct_nulls). SEMQ 1.0 cannot read a
    /// floor saved with them. With [`GateOptions::new`] it is `measure`.
    pub fn measure_for<D: AsRef<Diff>>(
        null_diffs: impl IntoIterator<Item = D>,
        checks: &GateOptions,
    ) -> Result<Floor> {
        // Hold every item until the call returns: an owned Diff dropped while
        // collecting pointers would free the handle the core is about to read.
        let diffs: Vec<D> = null_diffs.into_iter().collect();
        let ptrs: Vec<*const sys::semq_diff_t> =
            diffs.iter().map(|d| d.as_ref().as_ptr()).collect();
        let k = u32::try_from(ptrs.len())
            .map_err(|_| Error::invalid("measure takes fewer than 2^32 diffs"))?;
        let mut out: *mut sys::semq_floor_t = ptr::null_mut();
        let mut err = new_error();
        // SAFETY: `diffs` keeps every handle alive for the call; the array
        // holds `k` pointers; the out-pointers are live for the call.
        let status = unsafe {
            sys::semq_floor_measure_for(ptr_or_null(&ptrs), k, checks.checks(), &mut out, &mut err)
        };
        drop(diffs);
        check(status, &err, "measure")?;
        Self::from_raw(out, "measure")
    }

    /// Take ownership of a handle the core just produced.
    fn from_raw(ptr: *mut sys::semq_floor_t, operation: &'static str) -> Result<Floor> {
        let ptr = NonNull::new(ptr).ok_or_else(|| {
            Error::native(
                operation,
                sys::SEMQ_ERR_INTERNAL,
                None,
                None,
                "core returned a null floor",
            )
        })?;
        let guard = FreeOnDrop::new(ptr, sys::semq_floor_free);
        // SAFETY: the handle is live and owns the config it points to.
        let raw = unsafe { *sys::semq_floor_config(guard.ptr()) };
        let config = CodecConfig::from_raw(raw, operation)?;
        // SAFETY: the handle is live.
        let kind = unsafe { sys::semq_floor_id_kind(guard.ptr()) };
        let id_kind = IdKind::from_raw(kind).ok_or_else(|| {
            Error::native(
                operation,
                sys::SEMQ_ERR_INTERNAL,
                None,
                None,
                "floor has an unknown id kind",
            )
        })?;
        Ok(Floor {
            ptr: guard.release(),
            config,
            id_kind,
        })
    }

    pub(crate) fn as_ptr(&self) -> *const sys::semq_floor_t {
        self.ptr.as_ptr()
    }

    /// The config the nulls were measured under.
    pub fn config(&self) -> &CodecConfig {
        &self.config
    }

    /// The id kind of the nulls.
    pub fn id_kind(&self) -> IdKind {
        self.id_kind
    }

    /// The `state_id` of the reference every null was taken against.
    pub fn reference_id(&self) -> [u8; 32] {
        let mut out = [0u8; 32];
        // SAFETY: the handle is live and `out` has room for 32 bytes.
        unsafe { sys::semq_floor_reference_id(self.ptr.as_ptr(), out.as_mut_ptr()) };
        out
    }

    /// How many null diffs the floor was measured from.
    pub fn nulls(&self) -> u64 {
        // SAFETY: the handle is live.
        unsafe { sys::semq_floor_nulls(self.ptr.as_ptr()) }
    }

    /// Numerator of the admitted fraction of changed rows.
    pub fn changed_rows(&self) -> u64 {
        // SAFETY: the handle is live.
        unsafe { sys::semq_floor_changed_rows(self.ptr.as_ptr()) }
    }

    /// Denominator of the admitted fraction of changed rows.
    pub fn total_rows(&self) -> u64 {
        // SAFETY: the handle is live.
        unsafe { sys::semq_floor_total_rows(self.ptr.as_ptr()) }
    }

    /// Largest admitted p99 hamming distance over changed rows.
    pub fn hamming(&self) -> u64 {
        // SAFETY: the handle is live.
        unsafe { sys::semq_floor_hamming(self.ptr.as_ptr()) }
    }

    /// The largest hamming of any changed row of any null (`0` when no null
    /// changed a row). `None` for a floor without per-row data: one from
    /// [`new`](Self::new), [`measure`](Self::measure) or
    /// [`from_report`](Self::from_report), or read from JSON without it.
    pub fn max_hamming(&self) -> Option<u64> {
        // SAFETY: the handle is live.
        let v = unsafe { sys::semq_floor_max_hamming(self.ptr.as_ptr()) };
        (v != sys::SEMQ_NONE).then_some(v)
    }

    /// How many of the nulls were distinct states, from `1` to
    /// [`nulls`](Self::nulls): two nulls are the same when their candidates
    /// have the same `content_digest`. With `1` the per-row check adds no
    /// false alarm; when the nulls vary, each check can reject an unchanged
    /// rebuild with probability up to `1 / (nulls + 1)`. `None` for a floor
    /// without per-row data.
    pub fn distinct_nulls(&self) -> Option<u64> {
        // SAFETY: the handle is live.
        let v = unsafe { sys::semq_floor_distinct_nulls(self.ptr.as_ptr()) };
        (v != sys::SEMQ_NONE).then_some(v)
    }

    /// The report as plain data, ready for any serializer. It does not
    /// carry the per-row data; [`to_json`](Self::to_json) does.
    pub fn as_report(&self) -> FloorReport {
        FloorReport {
            version: FloorReport::VERSION.to_owned(),
            config: self.config,
            id_kind: self.id_kind.name().to_owned(),
            reference_id: hex(&self.reference_id()),
            nulls: self.nulls(),
            changed_rows: self.changed_rows(),
            total_rows: self.total_rows(),
            hamming: self.hamming(),
        }
    }

    /// The inverse of [`as_report`](Self::as_report), strictly: `version`
    /// must be [`FloorReport::VERSION`], `id_kind` `"u64"` or `"utf8"`,
    /// `reference_id` 64 hex characters, and the counts must satisfy
    /// [`new`](Self::new); otherwise `InvalidInput`. The result has no
    /// per-row data, because the report does not carry it: read a floor
    /// that has it with [`from_json`](Self::from_json).
    pub fn from_report(report: &FloorReport) -> Result<Floor> {
        if report.version != FloorReport::VERSION {
            return Err(Error::invalid(format!(
                "floor.version must be {:?}",
                FloorReport::VERSION
            )));
        }
        let id_kind = match report.id_kind.as_str() {
            "u64" => IdKind::U64,
            "utf8" => IdKind::Utf8,
            _ => return Err(Error::invalid("floor.id_kind must be \"u64\" or \"utf8\"")),
        };
        let reference_id = unhex32(&report.reference_id)
            .ok_or_else(|| Error::invalid("floor.reference_id must be 64 hex characters"))?;
        Floor::new(
            &report.config,
            id_kind,
            reference_id,
            report.nulls,
            report.changed_rows,
            report.total_rows,
            report.hamming,
        )
    }

    /// The floor's JSON form, written by the core: the
    /// [floor schema](https://the-semq-group.github.io/semq/reference/contracts/#floor-schema)
    /// with the per-row data when the floor records it, keys in schema order, no
    /// whitespace. The same bytes in every binding.
    pub fn to_json(&self) -> String {
        // SAFETY: the handle is live.
        let n = unsafe { sys::semq_floor_json_size(self.ptr.as_ptr()) };
        let mut out = vec![0u8; n as usize];
        let mut err = new_error();
        // SAFETY: `out` holds `n` writable bytes; the handle is live.
        let status =
            unsafe { sys::semq_floor_save(self.ptr.as_ptr(), out.as_mut_ptr(), n, &mut err) };
        check(status, &err, "save").expect("SEMQ core violated an infallible save contract");
        String::from_utf8(out).expect("SEMQ core wrote a floor that is not UTF-8")
    }

    /// Read a floor from its JSON form, by the core's rules: the schema's
    /// keys strictly (each at most once, integer counts, `max_hamming` and
    /// `distinct_nulls` optional), other keys ignored, and the rules of
    /// [`new`](Self::new). Anything else is `InvalidInput`.
    pub fn from_json(json: impl AsRef<[u8]>) -> Result<Floor> {
        let bytes = json.as_ref();
        let mut out: *mut sys::semq_floor_t = ptr::null_mut();
        let mut err = new_error();
        // SAFETY: `bytes` is readable for its length; the out-pointers are live.
        let status =
            unsafe { sys::semq_floor_load(bytes.as_ptr(), bytes.len() as u64, &mut out, &mut err) };
        check(status, &err, "load")?;
        Self::from_raw(out, "load")
    }

    #[allow(clippy::type_complexity)]
    fn fields(
        &self,
    ) -> (
        CodecConfig,
        IdKind,
        [u8; 32],
        u64,
        u64,
        u64,
        u64,
        Option<u64>,
        Option<u64>,
    ) {
        (
            self.config,
            self.id_kind,
            self.reference_id(),
            self.nulls(),
            self.changed_rows(),
            self.total_rows(),
            self.hamming(),
            self.max_hamming(),
            self.distinct_nulls(),
        )
    }
}

impl PartialEq for Floor {
    fn eq(&self, other: &Floor) -> bool {
        self.fields() == other.fields()
    }
}

impl Eq for Floor {}

impl Hash for Floor {
    fn hash<H: Hasher>(&self, state: &mut H) {
        self.fields().hash(state);
    }
}

/// `Floor(1 of 3 rows, hamming 1, from 3 nulls)`, the same text as the
/// other bindings.
impl fmt::Display for Floor {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        write!(
            f,
            "Floor({} of {} rows, hamming {}, from {} nulls)",
            self.changed_rows(),
            self.total_rows(),
            self.hamming(),
            self.nulls()
        )
    }
}

impl fmt::Debug for Floor {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        f.debug_struct("Floor")
            .field("config", &self.config)
            .field("id_kind", &self.id_kind.name())
            .field("reference_id", &hex(&self.reference_id()))
            .field("nulls", &self.nulls())
            .field("changed_rows", &self.changed_rows())
            .field("total_rows", &self.total_rows())
            .field("hamming", &self.hamming())
            .field("max_hamming", &self.max_hamming())
            .field("distinct_nulls", &self.distinct_nulls())
            .finish()
    }
}

/// A [`Floor`] as plain data: the `semq-floor/1` schema without a
/// serializer.
///
/// Serialized in this field order: `version` is `"semq-floor/1"`; `config`
/// is the same object a [`DiffReport`](crate::DiffReport) carries
/// (`{"operator": config.operator().name(), "dim": config.dim(),
/// config.operator().parameter_name(): config.parameter(), "rule_revision":
/// config.rule_revision()}`); `id_kind` is `"u64"` or `"utf8"`;
/// `reference_id` is 64 lowercase hex characters; the four counts are
/// integers. It does not carry the per-row data (`max_hamming`,
/// `distinct_nulls`): [`Floor::from_report`] is the strict inverse for a
/// floor without it, and [`Floor::to_json`] and [`Floor::from_json`] carry
/// it.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct FloorReport {
    /// The schema version, [`VERSION`](Self::VERSION).
    pub version: String,
    /// The config the nulls were measured under.
    pub config: CodecConfig,
    /// `"u64"` or `"utf8"`.
    pub id_kind: String,
    /// The reference's `state_id`, hex.
    pub reference_id: String,
    /// How many null diffs the floor was measured from.
    pub nulls: u64,
    /// Numerator of the admitted fraction of changed rows.
    pub changed_rows: u64,
    /// Denominator of the admitted fraction of changed rows.
    pub total_rows: u64,
    /// Largest admitted p99 hamming distance over changed rows.
    pub hamming: u64,
}

impl FloorReport {
    /// The schema version every report carries.
    pub const VERSION: &'static str = "semq-floor/1";
}

/// Exactly 64 hex digits (either case) as 32 bytes.
fn unhex32(text: &str) -> Option<[u8; 32]> {
    let digits = text.as_bytes();
    if digits.len() != 64 {
        return None;
    }
    let mut out = [0u8; 32];
    for (byte, pair) in out.iter_mut().zip(digits.chunks_exact(2)) {
        let hi = (pair[0] as char).to_digit(16)?;
        let lo = (pair[1] as char).to_digit(16)?;
        *byte = (hi * 16 + lo) as u8;
    }
    Some(out)
}