toktier 0.3.3

Rust API preview for TokTier's exact, stateful tokenization runtime
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
//! What version of its text semantics this binary actually carries.
//!
//! Three libraries under the reference engine decide where text is cut:
//! the `regex` crate's Unicode tables, the Oniguruma library `onig`
//! binds, and `unicode-segmentation`'s tables. Their correct behaviour is
//! defined by an evolving external standard, so a new package version of
//! one of them is not by itself a change of behaviour -- and a change of
//! behaviour is what can move ids. The certificate is about the second.
//!
//! The comparison has to happen here rather than in the build script.
//! A build script cannot link the libraries the crate being built links:
//! its own build dependencies are a separate resolution for the host,
//! possibly of other versions. So the build script hands over what it
//! knows -- the judged behaviour version of each unit, and whether the
//! package versions moved -- and this module asks the linked libraries
//! themselves.
//!
//! Fail-closed in every direction of the *reading*: a unit whose version
//! cannot be read here falls back to the build script's exact package
//! comparison; a record that names no unit at all leaves the whole core
//! on that exact comparison; and a package that resolves from somewhere
//! other than the judged one never reaches this module, because no probe
//! of a library's version can speak for a copy that is not the judged
//! one. What the reading decides changed in 0.3.0 (user rule of
//! 2026-10-14): a reference library that is not the judged one is a
//! label -- `reference_certification_reason` reads
//! `uncertified_reference`, the route is `supported_untested`, and the
//! judged recipe (`certified`) is not claimed -- while the accelerated
//! engines run. The reference route is for builds whose own sources do
//! not verify.

use std::sync::OnceLock;

use crate::diagnostics::{BehaviorVersion, ReferenceVersion};

/// Separators of the one-line values the build script emits. Both are
/// ASCII separators, so neither can appear in a version or a report.
const FIELD_SEPARATOR: char = '\u{1f}';
const RECORD_SEPARATOR: char = '\u{1e}';

/// The Unicode versions `regex` can be asked about, newest first. The
/// engine accepts `\p{Age=X}` for every version its tables carry, so the
/// first one that compiles is the version of those tables.
const AGE_CANDIDATES: &[&str] = &[
    "23.0", "22.0", "21.0", "20.0", "19.0", "18.0", "17.0", "16.0", "15.1", "15.0", "14.0", "13.0",
    "12.1", "12.0", "11.0", "10.0",
];

/// One unit, as the record judged it and as this binary reads it.
struct Reading {
    unit: String,
    judged: String,
    source: String,
    observed: Option<String>,
}

/// The judged behaviour versions, read from the record the build script
/// carried over.
fn judged_readings() -> Vec<Reading> {
    let recorded = env!("TOKTIER_RUST_API_JUDGED_BEHAVIOR");
    if recorded.is_empty() {
        return Vec::new();
    }
    recorded
        .split(FIELD_SEPARATOR)
        .filter_map(|record| {
            let mut fields = record.split(RECORD_SEPARATOR);
            let unit = fields.next()?;
            let judged = fields.next()?;
            let source = fields.next()?;
            if unit.is_empty() || judged.is_empty() {
                return None;
            }
            Some(Reading {
                observed: observe(unit),
                unit: unit.to_owned(),
                judged: judged.to_owned(),
                source: source.to_owned(),
            })
        })
        .collect()
}

/// The package moves the build script saw for one unit, if any.
fn drift_for(unit: &str) -> Option<&'static str> {
    let recorded = env!("TOKTIER_RUST_API_R2_DRIFT");
    recorded.split(FIELD_SEPARATOR).find_map(|record| {
        let (name, report) = record.split_once(RECORD_SEPARATOR)?;
        (name == unit).then_some(report)
    })
}

/// The version of one unit as the libraries this binary links report it.
///
/// `None` for a unit this release has no way to ask, which is the same
/// answer as a probe that failed: either way the build script's exact
/// comparison is what is left.
fn observe(unit: &str) -> Option<String> {
    match unit {
        // The tables are not versioned in the API, but the engine
        // accepts `\p{Age=X}` exactly for the versions they carry, so
        // the newest one that compiles is their version.
        "regex" => AGE_CANDIDATES
            .iter()
            .find(|age| regex::Regex::new(&format!(r"\p{{Age={age}}}")).is_ok())
            .map(|age| (*age).to_owned()),
        // Oniguruma publishes no Unicode version of its own; its library
        // version is what its own history ties those tables to, and the
        // binding reports the library it actually linked.
        "onig" => Some(onig::version()).filter(|version| !version.is_empty()),
        "unicode-segmentation" => {
            let (major, minor, patch) = unicode_segmentation::UNICODE_VERSION;
            Some(format!("{major}.{minor}.{patch}"))
        }
        _ => None,
    }
}

/// How a unit says which version of its tables this binary carries.
fn carries(unit: &str, observed: &str) -> String {
    match unit {
        "regex" => format!("carries Unicode {observed} (regex tables)"),
        "onig" => format!("links Oniguruma {observed}"),
        "unicode-segmentation" => format!("reads segmentation tables {observed}"),
        other => format!("reads {other} {observed}"),
    }
}

/// How a unit says that a package move left its tables where they were.
fn unchanged(unit: &str, version: &str) -> String {
    match unit {
        "regex" => format!("Unicode data version unchanged ({version})"),
        "onig" => format!("Oniguruma library version unchanged ({version})"),
        "unicode-segmentation" => format!("segmentation table version unchanged ({version})"),
        other => format!("{other} behaviour version unchanged ({version})"),
    }
}

/// How to see, on one's own text, what the accelerated engines would do
/// differently from the reference engine in this binary.
///
/// The command takes the measurement and records it; it is the same one
/// the `supported_untested` label points at, and it is the only thing
/// this crate offers here, because whether a particular text is affected
/// is a question about that text and nobody else can answer it. A crate
/// consumer has the library and not the binary, since Cargo does not
/// install a dependency's binaries, so the line says where the CLI comes
/// from and names the Python package's command of the same name as well.
const COMPARE_HINT: &str = "To compare the accelerated engine with this binary's reference \
                            engine on your own text, run `toktier-rust verify-local --engine \
                            cpu --family <family> --input <path>` where that CLI is \
                            installed (`cargo install --locked toktier` builds it; a crate \
                            dependency alone brings no binary), or `toktier verify-local` \
                            with the same options from the Python package";

/// What to offer where a reference library's tables are not the judged
/// ones: the accelerated engines run, labelled, so the comparison above
/// is available and worth making, and the `dependency_closure` line of
/// the same report carries the `cargo update --precise` command that
/// brings the library back onto the judged behaviour version.
const ALIGN_HINT: &str = "the `dependency_closure` line of this same report carries the `cargo \
                          update --precise` command for each package it names, to run in the \
                          workspace that owns the governing lockfile, which brings the library \
                          back onto the judged behaviour version";

fn readings() -> &'static Vec<Reading> {
    static READINGS: OnceLock<Vec<Reading>> = OnceLock::new();
    READINGS.get_or_init(judged_readings)
}

/// How the certified core of this build compares with the judged one:
/// `verified`, an `unlocated:` line, or a `mismatched:` line saying what
/// differs and what aligns it.
///
/// Since 0.3.0 this is the build script's reading of the workspace
/// members -- this crate and its siblings resolved as the judged copies,
/// or not at all where an optional feature is off -- and of whether a
/// comparison could be made. The reference libraries, probed here or
/// read by crate version, are reported in `reference_versions`,
/// `behavior_versions` and `reference_certification_reason` instead:
/// they label the route, they do not decide it.
pub(crate) fn core_closure() -> &'static str {
    env!("TOKTIER_RUST_API_CORE_CLOSURE_STATIC")
}

pub(crate) fn core_closure_verified() -> bool {
    core_closure() == "verified"
}

/// Whether the closure says this build's own sources are the judged ones
/// as far as it can tell: verified, or no lockfile to compare against.
/// This is what admits the fast CPU engine by source identity; a
/// workspace member resolved from another copy or version is a source
/// that does not verify, and so is a record this build cannot read.
pub(crate) fn sources_closure_admits() -> bool {
    let core = core_closure();
    core == "verified" || core.starts_with("unlocated")
}

/// What the probed units say about the tables this binary carries, when
/// they are not the judged ones: one sentence naming the unit, both
/// versions and what follows, or `None` when every probed unit agrees.
///
/// Taken apart from where the readings come from so that each state can
/// be exercised: a unit whose tables moved; a unit that cannot be read
/// while its packages moved; and everything agreeing.
fn probe_divergence(
    readings: &[Reading],
    drift: &dyn Fn(&str) -> Option<&'static str>,
) -> Option<String> {
    for reading in readings {
        match &reading.observed {
            Some(observed) if observed != &reading.judged => {
                return Some(format!(
                    "the reference engine in this binary {} where the certified evidence was \
                     taken on {}; some code points may tokenize differently; the accelerated \
                     engines run, the route is labelled supported_untested and \
                     reference_certification_reason reads uncertified_reference. {ALIGN_HINT}. \
                     {COMPARE_HINT}",
                    carries(&reading.unit, observed),
                    reading.judged
                ));
            }
            Some(_) => {}
            None => {
                if let Some(moved) = drift(&reading.unit) {
                    return Some(format!(
                        "the behaviour version of {} could not be read in this binary, so the \
                         package versions are compared instead: {moved}; the accelerated \
                         engines run, the route is labelled supported_untested and \
                         reference_certification_reason reads uncertified_reference. \
                         {ALIGN_HINT}",
                        reading.unit
                    ));
                }
            }
        }
    }
    None
}

fn probe_status() -> Option<&'static str> {
    static STATUS: OnceLock<Option<String>> = OnceLock::new();
    STATUS
        .get_or_init(|| probe_divergence(readings(), &drift_for))
        .as_deref()
}

/// What this build compiles that the judged build did not, where the
/// difference is reported rather than gating.
///
/// Two kinds of thing arrive here: packages outside the certified core,
/// and core packages whose package version moved while the behaviour
/// version this binary reads stayed where the evidence was taken. Both
/// are named, with the commands that align them; neither is called
/// harmless.
pub(crate) fn dependency_advisory() -> Option<&'static str> {
    static ADVISORY: OnceLock<Option<String>> = OnceLock::new();
    ADVISORY
        .get_or_init(|| {
            let mut parts = Vec::new();
            let outside = env!("TOKTIER_RUST_API_DEPENDENCY_ADVISORY");
            if !outside.is_empty() {
                parts.push(outside.to_owned());
            }
            if let Some(divergence) = probe_status() {
                parts.push(divergence.to_owned());
            }
            for reading in readings() {
                let Some(observed) = &reading.observed else {
                    continue;
                };
                if observed != &reading.judged {
                    continue;
                }
                if let Some(drift) = drift_for(&reading.unit) {
                    parts.push(format!(
                        "{drift}: {}; the certificate holds. {COMPARE_HINT}",
                        unchanged(&reading.unit, observed)
                    ));
                }
            }
            (!parts.is_empty()).then(|| parts.join(". "))
        })
        .as_deref()
}

/// The reference libraries judged by crate version -- the embedded
/// reference engine, the Unicode property data, the normalization and
/// SentencePiece tables -- each with the version this build resolved,
/// the judged version, and the certified reference versions. The build
/// script read them from the governing lockfile; this module only
/// renders the record.
fn reference_readings() -> &'static Vec<ReferenceVersion> {
    static READINGS: OnceLock<Vec<ReferenceVersion>> = OnceLock::new();
    READINGS.get_or_init(|| {
        let recorded = env!("TOKTIER_RUST_API_REFERENCE_VERSIONS");
        if recorded.is_empty() {
            return Vec::new();
        }
        recorded
            .split(FIELD_SEPARATOR)
            .filter_map(|record| {
                let mut fields = record.split(RECORD_SEPARATOR);
                let package = fields.next()?;
                let judged = fields.next()?;
                let resolved = fields.next()?;
                let certified = fields.next()?;
                let status = fields.next()?;
                if package.is_empty() || judged.is_empty() {
                    return None;
                }
                Some(ReferenceVersion {
                    package: package.to_owned(),
                    judged: judged.to_owned(),
                    resolved: (!resolved.is_empty()).then(|| resolved.to_owned()),
                    certified_versions: certified
                        .split(',')
                        .filter(|version| !version.is_empty())
                        .map(str::to_owned)
                        .collect(),
                    certified: status == "certified",
                })
            })
            .collect()
    })
}

/// The reference libraries, as a reader of `doctor` sees them.
pub(crate) fn reference_facts() -> Vec<ReferenceVersion> {
    reference_readings().clone()
}

/// Why the reference libraries of this build are not the certified ones,
/// in the word the Python facade's `doctor` uses for the same state:
/// `None` when every reference library resolves at a certified reference
/// version and every probed unit reads the judged behaviour version,
/// `uncertified_reference` otherwise -- a version outside the set, a copy
/// from another source, tables at another Unicode version, or a build
/// whose governing lockfile could not be located, which is answered
/// fail-closed. Since 0.3.0 this is a label on a running route, not a
/// refusal.
pub(crate) fn reference_certification_reason() -> Option<&'static str> {
    // Three readings, any one of which is enough: a crate-version unit
    // outside its certified reference versions (or unlocated); a probed
    // unit whose tables are not the judged ones; and a reference library
    // the build script could not speak for at all -- a probed unit's
    // package from another source, or a package of a unit the record
    // disagrees about -- which has no record in the first two.
    let unspoken_for = !env!("TOKTIER_RUST_API_REFERENCE_UNCERTIFIED").is_empty();
    (reference_readings()
        .iter()
        .any(|reading| !reading.certified)
        || probe_status().is_some()
        || unspoken_for)
        .then_some("uncertified_reference")
}

/// The behaviour version of each unit, as a reader of `doctor` sees it.
pub(crate) fn facts() -> Vec<BehaviorVersion> {
    readings()
        .iter()
        .map(|reading| BehaviorVersion {
            unit: reading.unit.clone(),
            observed: reading.observed.clone(),
            judged: reading.judged.clone(),
            source: reading.source.clone(),
        })
        .collect()
}

#[cfg(test)]
mod tests {
    use super::{
        carries, drift_for, observe, probe_divergence, readings, unchanged, Reading,
        AGE_CANDIDATES, FIELD_SEPARATOR, RECORD_SEPARATOR,
    };

    fn reading(unit: &str, judged: &str, observed: Option<&str>) -> Reading {
        Reading {
            unit: unit.to_owned(),
            judged: judged.to_owned(),
            source: "probe:regex-age".to_owned(),
            observed: observed.map(str::to_owned),
        }
    }

    fn no_drift(_: &str) -> Option<&'static str> {
        None
    }

    /// The three states of the probe reading, each asked directly. Since
    /// 0.3.0 a divergence is a label: the sentence says the engines run and
    /// what the label is, and carries the alignment and comparison hints.
    #[test]
    fn the_probe_reading_has_three_states() {
        // Tables that moved: named, with both versions and what follows.
        let moved = probe_divergence(&[reading("regex", "16.0", Some("17.0"))], &no_drift)
            .expect("a divergence");
        assert!(
            moved.starts_with(
                "the reference engine in this binary carries Unicode 17.0 (regex tables) where \
                 the certified evidence was taken on 16.0; some code points may tokenize \
                 differently; the accelerated engines run, the route is labelled \
                 supported_untested and reference_certification_reason reads \
                 uncertified_reference. the `dependency_closure` line"
            ),
            "{moved}"
        );
        assert!(moved.contains("verify-local --engine cpu"), "{moved}");
        for word in ["held on the reference route", "admits no accelerated"] {
            assert!(!moved.contains(word), "{moved}");
        }
        // A unit that cannot be read falls back to the package comparison
        // the build script made, and only reports when that comparison
        // found something.
        assert_eq!(
            probe_divergence(&[reading("regex", "16.0", None)], &no_drift),
            None
        );
        let unread = probe_divergence(&[reading("regex", "16.0", None)], &|unit| {
            (unit == "regex").then_some("regex 1.13.1 -> 1.14.0")
        })
        .expect("a divergence");
        assert!(
            unread.starts_with(
                "the behaviour version of regex could not be read in this binary, so the \
                 package versions are compared instead: regex 1.13.1 -> 1.14.0; the \
                 accelerated engines run"
            ),
            "{unread}"
        );
        // Everything agreeing.
        assert_eq!(
            probe_divergence(
                &[
                    reading("regex", "16.0", Some("16.0")),
                    reading("onig", "6.9.10", Some("6.9.10")),
                ],
                &no_drift,
            ),
            None
        );
    }

    /// A record that names no unit at all has nothing to probe, and the
    /// closure answer is the build script's alone.
    #[test]
    fn a_record_with_no_units_has_nothing_to_report() {
        assert_eq!(probe_divergence(&[], &no_drift), None);
    }

    /// Inside this workspace every reference library resolves at its
    /// judged version, the record names the embedded reference engine
    /// among them, and there is no reason to report.
    #[test]
    fn every_reference_library_resolves_at_a_certified_version_here() {
        let readings = super::reference_facts();
        assert!(!readings.is_empty(), "no reference library was recorded");
        assert!(readings
            .iter()
            .any(|reading| reading.package == "tokenizers"));
        for reading in &readings {
            assert_eq!(
                reading.resolved.as_deref(),
                Some(reading.judged.as_str()),
                "{} does not resolve at its judged version here; a build of this workspace \
                 with a target directory outside the tree locates no governing lockfile \
                 unless TOKTIER_CARGO_LOCK is set to the absolute path of the workspace's \
                 own Cargo.lock",
                reading.package
            );
            assert!(reading.certified_versions.contains(&reading.judged));
            assert!(reading.certified, "{} is not certified", reading.package);
        }
        assert_eq!(super::reference_certification_reason(), None);
    }

    /// The record and the libraries agree in this workspace. The judged
    /// values are written by `tools/generate_judged_closure.py` from the
    /// vendored sources of the same packages this build links, so a lock
    /// that moved without the record being regenerated is red here.
    #[test]
    fn every_judged_unit_reads_the_way_the_record_says() {
        let readings = readings();
        assert!(!readings.is_empty(), "no behaviour unit was recorded");
        for reading in readings {
            let observed = reading
                .observed
                .as_deref()
                .unwrap_or_else(|| panic!("{} could not be read", reading.unit));
            assert_eq!(
                observed, reading.judged,
                "{} reads {observed} and the record judges {}",
                reading.unit, reading.judged
            );
        }
    }

    /// The three units this release knows how to ask, and nothing else.
    #[test]
    fn a_unit_this_release_cannot_ask_reads_as_unreadable() {
        assert!(observe("regex").is_some());
        assert!(observe("onig").is_some());
        assert!(observe("unicode-segmentation").is_some());
        assert!(observe("unicode_categories").is_none());
        assert!(observe("").is_none());
    }

    /// The age probe finds the newest version the tables carry, not the
    /// newest one it asks about.
    #[test]
    fn the_age_probe_answers_from_the_tables_rather_than_the_candidates() {
        let observed = observe("regex").expect("a regex age");
        assert!(AGE_CANDIDATES.contains(&observed.as_str()));
        assert!(
            regex::Regex::new(&format!(r"\p{{Age={observed}}}")).is_ok(),
            "the answer does not compile"
        );
        let newer = AGE_CANDIDATES
            .iter()
            .take_while(|age| **age != observed)
            .collect::<Vec<_>>();
        for age in newer {
            assert!(
                regex::Regex::new(&format!(r"\p{{Age={age}}}")).is_err(),
                "{age} compiles and is newer than the answer"
            );
        }
    }

    /// The unit sentences name the unit and both sides, and never say
    /// that a package outside the reading cannot matter.
    #[test]
    fn the_unit_sentences_say_which_tables_and_which_version() {
        assert_eq!(
            carries("regex", "17.0"),
            "carries Unicode 17.0 (regex tables)"
        );
        assert_eq!(carries("onig", "6.9.11"), "links Oniguruma 6.9.11");
        assert_eq!(
            unchanged("regex", "16.0"),
            "Unicode data version unchanged (16.0)"
        );
        assert_eq!(
            unchanged("a-new-unit", "1.0"),
            "a-new-unit behaviour version unchanged (1.0)"
        );
    }

    /// The drift line is read by unit, and a unit that is not in it has
    /// no drift rather than the first one's.
    #[test]
    fn the_drift_line_is_read_by_unit() {
        // The separators are the ones the build script writes.
        assert_eq!(FIELD_SEPARATOR, '\u{1f}');
        assert_eq!(RECORD_SEPARATOR, '\u{1e}');
        // This workspace's own build has no drift at all, which is what
        // a build inside the source workspace should read.
        assert!(drift_for("regex").is_none());
        assert!(drift_for("a-unit-that-is-not-recorded").is_none());
    }
}