toktier 0.3.2

Rust API preview for TokTier's exact, stateful tokenization runtime
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
/// Runtime certification policy.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)]
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
#[non_exhaustive]
pub enum Policy {
    /// Everything [`Policy::Certified`] admits, and in addition a device
    /// or compiler toolchain the certification campaigns have not judged,
    /// as long as the shipped kernel loads and runs on it. Since 0.3.2
    /// this includes, for the GPU route, a build from the judged sources
    /// whose Rust compiler or build flags no judged row records. Such a
    /// route is labelled [`Certification::Supported`] rather than
    /// certified, and [`Certification::LocallyVerified`] once a local
    /// check has compared it against the reference engine on this machine
    /// with this build.
    ///
    /// The default since 0.2.6. What it does **not** admit is any
    /// disagreement inside the certified core: sources that are not the
    /// judged ones, a sibling crate that is not the judged copy or a
    /// dependency closure that could not be compared, a reference library
    /// outside its certified versions (for the GPU route), a fatbin that
    /// does not match its digest, or a world-writable compiler component
    /// still routes to the reference engine.
    #[default]
    Supported,
    /// Admit only evidence-bound accelerated routes; fall back exactly.
    /// For the GPU route: a device architecture or an NVCC triple the
    /// registry has not judged is not eligible here, and neither is a
    /// build whose Rust compiler or build flags are not a judged recipe
    /// ([`ReasonCode::GpuBuildRecipeUnjudged`]). Since 0.3.0 the fast CPU
    /// route is admitted by its source identity under this policy too,
    /// labelled [`Certification::Supported`] when the Rust compiler or
    /// feature recipe is not a judged one; only a build whose sources do
    /// not verify routes to the reference engine.
    Certified,
    /// Force the frozen Hugging Face reference implementation.
    Reference,
    /// Require at least one certified accelerated backend at construction.
    /// Per-input safety guards may still fall back to the reference.
    RequireAccelerated,
    /// Permit explicitly requested experimental delivery while labelling it.
    Experimental,
}

impl Policy {
    /// Whether this policy admits a device or toolchain no campaign has
    /// judged, on a build whose own certified core is intact. Only the
    /// GPU routes ask; a CPU-only build compiles the method unused.
    #[cfg_attr(not(any(feature = "prebuilt-gpu", feature = "jit")), allow(dead_code))]
    pub(crate) fn admits_unjudged_device(self) -> bool {
        matches!(self, Self::Supported | Self::RequireAccelerated)
    }
}

/// Requested execution device.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)]
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
#[non_exhaustive]
pub enum Device {
    #[default]
    Auto,
    Cpu,
    Cuda(u32),
}

/// CUDA kernel delivery preference.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)]
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
#[non_exhaustive]
pub enum GpuDelivery {
    #[default]
    Auto,
    Prebuilt,
    Jit,
    Disabled,
}

/// Backend that actually produced token IDs.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
#[non_exhaustive]
pub enum Backend {
    HuggingFace,
    FastCpu,
    Gpu,
}

impl Backend {
    pub(crate) fn from_internal(value: &str) -> Self {
        match value {
            "gpu" => Self::Gpu,
            "fast_cpu" => Self::FastCpu,
            _ => Self::HuggingFace,
        }
    }
}

/// Stable route/fallback reason identifier.
///
/// The run-time variants are this crate's spelling of the frozen `R_*`
/// namespace in `docs/contracts/routing.md` Section 5: the router records
/// those codes in its own ledger, and the ones reachable from a Rust
/// execution are named here rather than in a second vocabulary. The
/// plan-time variants describe admission decisions this crate makes for
/// itself and are not claimed to be the same codes; they are documented
/// as their own vocabulary in `docs/contracts/routing.md` Section 5.6,
/// one of them having no Python counterpart at all.
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
#[non_exhaustive]
pub enum ReasonCode {
    /// `R_INPUT_BELOW_GPU_THRESHOLD`
    InputBelowGpuThreshold,
    /// `R_INPUT_ADDED_TOKEN`
    InputAddedToken,
    /// `R_INPUT_GUARD_ROUTED`
    InputGuardRouted,
    /// `R_EXEC_FAULT`
    ExecutionFault,
    /// `R_INPUT_POSTPROCESS_ROUTED`
    InputPostprocessRouted,
    /// `R_SESSION_NO_SAFE_CUT`
    SessionNoSafeCut,
    /// `R_INVALID_PRIOR_STATE`
    ///
    /// A stored session tail did not describe itself consistently, so
    /// the accumulated text was re-encoded from the reference. Named in
    /// the frozen namespace since 0.2.7; before that it travelled as an
    /// `Other` string that was in neither vocabulary.
    InvalidPriorState,
    /// `R_CHUNK_NO_CUT`: a chunk-parallel encode found no certified
    /// stable boundary in the document, so it was encoded serially on
    /// the same route. Added in 0.3.0.
    ChunkNoCut,
    /// `R_CHUNK_FAMILY_UNSUPPORTED`: chunk-parallel encode was asked for
    /// a tokenizer it does not admit, and the document was encoded
    /// serially on the plain route; `chunked::Admission::refusal` names
    /// the reason. Added in 0.3.0.
    ChunkFamilyUnsupported,
    /// `R_CHUNK_ROUTE_NOT_FAST_CPU`: chunk-parallel encode was asked for
    /// an input whose route does not start at the corrected CPU engine
    /// (a GPU-headed plan at or above its crossover, or a route without
    /// the engine), so it was encoded serially through the normal route.
    /// Added in 0.3.0.
    ChunkRouteNotFastCpu,
    /// A GPU route was wanted and this runtime could not offer one
    /// here: the build carries no `prebuilt-gpu` feature, or the GPU
    /// runtime failed to open on this machine.
    GpuUnavailable,
    /// A GPU route was wanted and is not certified: either this build
    /// is not certified for one, or the artifact has no eligible GPU
    /// identity.
    GpuUncertified,
    /// A GPU route was wanted under [`Policy::Certified`] on a build
    /// whose source identities are judged and whose toolchain or build
    /// flags no judged row records. That policy admits the GPU route only
    /// on the judged recipe; [`Policy::Supported`] runs it labelled
    /// [`Certification::Supported`]. Recorded beside
    /// [`ReasonCode::GpuUncertified`]. Added in 0.3.2.
    GpuBuildRecipeUnjudged,
    /// The fast CPU route was refused: no certified repair row for this
    /// artifact, or an engine status, artifact or source binding that
    /// does not match the registry. Since 0.3.0 the compiler is not part
    /// of this refusal: the judged engine sources built by another rustc
    /// are admitted under every policy but [`Policy::Reference`] and
    /// labelled [`Certification::Supported`].
    CpuUncertified,
    /// The artifact is an examined one, not a certified one: its pipeline
    /// capability is the one a certified family holds and its added-token
    /// table no campaign has judged (`docs/support-matrix.md`, Status
    /// vocabulary). The fast CPU engine runs it with its own repair
    /// parameters and the route is labelled [`Certification::Supported`];
    /// under [`Policy::Certified`] the route is refused and this reason
    /// stands beside [`ReasonCode::CpuUncertified`]. Added after 0.3.0.
    ExaminedArtifact,
    /// This build's own source identity is not in the shipped
    /// `runtime_builds` register, or one of this crate's own siblings
    /// resolves from another copy or version than the judged one, so no
    /// accelerated route is admitted whatever the artifact is. Since
    /// 0.3.0 the compiler, the build flags and the versions of the
    /// reference libraries are not part of this reason under any policy:
    /// a build from judged sources admits the fast CPU route and labels
    /// it.
    RuntimeBuildUncertified,
    /// [`Policy::Reference`] was asked for, so no accelerated option
    /// was considered.
    ReferenceRequested,
    /// A reason the router named that this release has no frozen code
    /// for. The string is the router's own token, passed through rather
    /// than mapped onto a code that does not mean it.
    Other(String),
}

/// The frozen codes for the two reasons the session repair names in its
/// path rather than in a ledger entry.
///
/// The router's crate carries no constant for either, because no router
/// path records them; these are the spelling on the Rust face, and they
/// are the same strings `docs/contracts/routing.md` Section 5.2 lists.
pub(crate) const R_SESSION_NO_SAFE_CUT: &str = "R_SESSION_NO_SAFE_CUT";
pub(crate) const R_INVALID_PRIOR_STATE: &str = "R_INVALID_PRIOR_STATE";

impl ReasonCode {
    /// The reason behind one routed execution, from the code the router
    /// recorded for it. Read rather than inferred, so this answer and the
    /// ledger cannot come to disagree.
    pub(crate) fn from_ledger_code(code: &str) -> Self {
        match code {
            toktier_routing_core::R_INPUT_BELOW_GPU_THRESHOLD => Self::InputBelowGpuThreshold,
            toktier_routing_core::R_INPUT_ADDED_TOKEN => Self::InputAddedToken,
            toktier_routing_core::R_INPUT_GUARD_ROUTED => Self::InputGuardRouted,
            toktier_routing_core::R_EXEC_FAULT => Self::ExecutionFault,
            toktier_routing_core::R_INPUT_POSTPROCESS_ROUTED => Self::InputPostprocessRouted,
            R_SESSION_NO_SAFE_CUT => Self::SessionNoSafeCut,
            R_INVALID_PRIOR_STATE => Self::InvalidPriorState,
            toktier_routing_core::R_CHUNK_NO_CUT => Self::ChunkNoCut,
            toktier_routing_core::R_CHUNK_FAMILY_UNSUPPORTED => Self::ChunkFamilyUnsupported,
            toktier_routing_core::R_CHUNK_ROUTE_NOT_FAST_CPU => Self::ChunkRouteNotFastCpu,
            other => Self::Other(other.to_owned()),
        }
    }
}

/// Certification status attached to one admitted route.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
#[non_exhaustive]
pub enum Certification {
    Certified,
    CertifiedSource,
    /// The engines are the judged ones and they run here, but this
    /// device or compiler toolchain is outside the judged list -- a GPU
    /// architecture or NVCC triple no campaign ran on, or, since 0.3.0,
    /// a Rust compiler or feature recipe no campaign built the fast CPU
    /// engine with -- so no campaign has measured this combination.
    /// An unjudged device is admitted under [`Policy::Supported`] and
    /// routed to the reference engine under `Policy::Certified`; an
    /// unjudged compiler or recipe for the fast CPU route is admitted
    /// under every policy but `Policy::Reference`, and for the GPU route
    /// (since 0.3.2) under `Policy::Supported` and
    /// `Policy::RequireAccelerated`, on a judged device too. The fast CPU route of
    /// an examined artifact carries this label too, on every build: the
    /// engine is the judged one and the artifact's added-token table is
    /// not one a campaign has judged ([`ReasonCode::ExaminedArtifact`]).
    Supported,
    /// A `Supported` route that a local check has compared against this
    /// binary's reference engine on this machine. The record is not a
    /// certificate: it says what was compared, by whom, and on what, and
    /// it expires when the driver, toolchain, kernel or source identity
    /// changes.
    LocallyVerified,
    Experimental,
    Reference,
}

/// Immutable route selected at tokenizer construction.
#[derive(Debug, Clone, PartialEq, Eq)]
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
#[non_exhaustive]
pub struct RoutePlan {
    pub family: String,
    pub artifact_sha256: String,
    pub backends: Vec<Backend>,
    pub gpu_min_bytes: u64,
    pub certification: Certification,
    pub reasons: Vec<ReasonCode>,
    /// How `add_special_tokens` is served for this artifact, read from
    /// the post-processor of the document the reference engine executes:
    ///
    /// - `"identity"`: the post-processor adds nothing to a single
    ///   sequence (none, `ByteLevel`, or a template that is the sequence
    ///   alone), so a request with special tokens runs the plain route;
    /// - `"prefix"`: a template that places one special token before the
    ///   sequence, so the plain route's core stream is returned behind
    ///   [`RoutePlan::special_token_prefix`];
    /// - `"reference"`: any other post-processor, or a construction
    ///   check that did not hold, so a request with special tokens is
    ///   encoded by the reference engine (`R_INPUT_POSTPROCESS_ROUTED`).
    ///
    /// Added in 0.3.2.
    pub postprocessor_class: String,
    /// The ids placed before the core stream when `postprocessor_class`
    /// is `"prefix"`; empty otherwise. Added in 0.3.2.
    pub special_token_prefix: Vec<u32>,
    /// `None` when the construction check of the class held or was not
    /// needed. Otherwise which probe text answered differently on the
    /// reference engine (with and without special tokens), the reason the
    /// class reads `"reference"`. Added in 0.3.2.
    pub postprocessor_self_check: Option<String>,
    /// `None` when the route segments the artifact's added-token literals
    /// (each text segment runs on the plain route, the literals are
    /// emitted by id). Otherwise why it does not -- an artifact outside
    /// the served class, a construction self-check of the segmenter that
    /// answered differently from the reference (session repair is then
    /// off as well), or a reference-only route -- and a literal-bearing
    /// input is encoded whole by the reference. Added in 0.3.2.
    pub literal_segmentation: Option<String>,
}

/// Facts for one completed encode or append operation.
#[derive(Debug, Clone, PartialEq, Eq)]
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
#[non_exhaustive]
pub struct ExecutionFacts {
    pub backend: Backend,
    pub path: String,
    pub source: Option<String>,
    pub input_bytes: u64,
    pub certification: Certification,
    /// Why this execution ran where it did, when a routing decision
    /// moved it off the first admitted backend. `None` means the
    /// admitted route ran and there was nothing to record; why the
    /// admitted route is what it is belongs to
    /// [`RoutePlan::reasons`](crate::RoutePlan), not to one execution.
    pub reason: Option<ReasonCode>,
}

/// Build/runtime facts that can be emitted by a control plane without
/// parsing human-readable output.
#[derive(Debug, Clone, PartialEq, Eq)]
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
#[non_exhaustive]
pub struct DoctorFacts {
    pub crate_version: String,
    pub oracle: String,
    pub registry_verified: bool,
    pub python_required: bool,
    pub sqlite_compiled: bool,
    pub prebuilt_gpu_compiled: bool,
    pub jit_compiled: bool,
    /// Whether this build can acquire an artifact over the network. The
    /// `network` feature is not in the default set, so a build that has
    /// to fetch and does not carry it answers `NETWORK_DISABLED`.
    pub network_compiled: bool,
    pub runtime_build: RuntimeBuildFacts,
    pub cuda: Option<CudaFacts>,
}

/// Exact build identity embedded by the Python-free Rust facade.
#[derive(Debug, Clone, PartialEq, Eq)]
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
#[non_exhaustive]
pub struct RuntimeBuildFacts {
    pub source_digest: String,
    pub fast_cpu_source_digest: String,
    pub native_host_source_digest: String,
    pub toolchain: String,
    pub build_flags: Vec<String>,
    /// How the whole compiled closure of this build compares with the
    /// judged one (three-way shape unchanged since 0.2.4). Since 0.2.6
    /// this is a reading, not a gate: `certified` looks at
    /// `core_closure`. Readers who want the 0.2.4 and 0.2.5 rule can
    /// require `dependency_closure == "verified"` themselves.
    pub dependency_closure: String,
    /// Why the build flags of this build are not the judged ones, when a
    /// judged entry agrees with it on every other axis. `None` when the
    /// flags match, or when something else is the disagreement.
    pub build_flag_divergence: Option<String>,
    /// `true` when the certified core of this build -- TokTier's own
    /// crates and the reference libraries beneath them -- is the one the
    /// certification evidence was taken on, together with the source
    /// identities, toolchain and build flags of a judged entry: the
    /// judged recipe, built from the judged sources. Packages outside
    /// that core are compared as well and reported in
    /// `dependency_advisory`; they do not decide this field. Since 0.3.0
    /// this is not the only state that admits the fast CPU route:
    /// `source_identity_judged` with the core verified admits it under
    /// `Policy::Supported` as `Certification::Supported`. Since 0.3.2 the
    /// same holds for the GPU route under `Policy::Supported` and
    /// `Policy::RequireAccelerated` when, in addition, every reference
    /// library is at a certified version and the closure is `verified`.
    pub certified: bool,
    /// `true` when the three source identities of this build -- the
    /// facade, the fast CPU engine and the native host -- are those of
    /// a judged entry in the shipped `runtime_builds` register, whatever
    /// compiler and flags built them.
    pub source_identity_judged: bool,
    /// `true` when a judged entry with this build's source identities
    /// also records this build's toolchain and build flags: the recipe
    /// the campaign built, not only the sources it built from.
    pub recipe_judged: bool,
    /// Why the compiler of this build is not a judged one, when a judged
    /// entry agrees with it on the source identities. `None` when the
    /// toolchain is recorded for these sources, or when the sources
    /// themselves are not judged.
    pub toolchain_divergence: Option<String>,
    /// The reference libraries judged by crate version -- the embedded
    /// reference engine, the Unicode property data, the normalization
    /// and SentencePiece tables -- each with the version this build
    /// resolved, the judged version and the certified reference
    /// versions.
    pub reference_versions: Vec<ReferenceVersion>,
    /// `None` when every reference library resolves at a certified
    /// reference version and every probed unit reads the judged behaviour
    /// version; `"uncertified_reference"` otherwise, the word the Python
    /// facade's `doctor` uses for the same state. A build in that state
    /// runs the accelerated engines labelled `supported_untested` and is
    /// not `certified`; the Unicode tables it reads may not be the judged
    /// ones (`dependency_advisory` says which and what aligns them).
    pub reference_certification_reason: Option<String>,
    /// The same comparison as `dependency_closure`, restricted to what
    /// decides admission by closure: this crate's own siblings, resolved
    /// as the judged copies or not at all where an optional feature is
    /// off. Anything other than `"verified"` here holds `certified` at
    /// `false` and says what differs and what aligns it; a `mismatched`
    /// line also closes the accelerated routes, an `unlocated` line does
    /// not. Since 0.3.0 the reference libraries are reported in
    /// `reference_versions`, `behavior_versions` and
    /// `reference_certification_reason` rather than here.
    pub core_closure: String,
    /// What this build compiles that the judged build did not, where the
    /// difference is reported rather than gating: packages outside the
    /// certified core, and core packages whose package version moved
    /// while the behaviour version this binary reads did not. `None`
    /// when there is nothing to report. It never says that the packages
    /// it names cannot influence behaviour; it says which packages the
    /// certificate covers and reports the rest.
    pub dependency_advisory: Option<String>,
    /// The version of the text semantics this binary carries, per unit,
    /// beside the version the evidence was taken on.
    pub behavior_versions: Vec<BehaviorVersion>,
}

/// The version of one text-semantics unit, as this binary reads it and
/// as the certification evidence recorded it.
///
/// A unit is the thing whose version can move ids: the Unicode tables a
/// regex engine carries, the Oniguruma library a binding links, the
/// segmentation tables a crate ships. `observed` is `None` for a unit
/// this binary has no way to ask, in which case the package versions of
/// that unit are compared exactly instead.
#[derive(Debug, Clone, PartialEq, Eq)]
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
#[non_exhaustive]
pub struct BehaviorVersion {
    pub unit: String,
    pub observed: Option<String>,
    pub judged: String,
    pub source: String,
}

/// One reference library judged by crate version, as this build
/// resolved it and as the certification evidence recorded it.
///
/// `resolved` is the version or versions this build's governing lockfile
/// reaches, `None` when no lockfile could be located; `certified` is
/// whether that resolution is one the certified reference versions
/// speak for.
#[derive(Debug, Clone, PartialEq, Eq)]
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
#[non_exhaustive]
pub struct ReferenceVersion {
    pub package: String,
    pub judged: String,
    pub resolved: Option<String>,
    pub certified_versions: Vec<String>,
    pub certified: bool,
}

/// Non-invasive CUDA probe result. Probe failure is data, not a panic.
#[derive(Debug, Clone, PartialEq, Eq)]
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
#[non_exhaustive]
pub struct CudaFacts {
    pub device_ordinal: u32,
    pub available: bool,
    pub architecture: Option<String>,
    /// The CUDA driver API version the device was opened through.
    ///
    /// An environment fact, not a certificate premise: it is reported
    /// because a reader wants to know it, and no certification claim
    /// rests on the number. Where a registry row binds a driver floor,
    /// that floor is checked separately as a precondition for the
    /// kernel loading at all.
    pub driver_api_version: Option<i32>,
    pub error: Option<String>,
}

/// Public snapshot of the native state-store counters.
#[derive(Debug, Clone, PartialEq)]
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
#[non_exhaustive]
pub struct StoreStats {
    pub format: String,
    pub session_count: u64,
    pub node_count: u64,
    pub puts: u64,
    pub appends: u64,
    pub forks: u64,
    pub lookups: u64,
    pub lookup_hits: u64,
    pub lookup_misses: u64,
    pub checksum_rejects: u64,
    pub sessions_evicted: u64,
    pub import_rejects: u64,
    pub revision_conflicts: u64,
    pub path_counts: BTreeMap<String, u64>,
}

impl From<toktier_store_core::StatsSnapshot> for StoreStats {
    fn from(value: toktier_store_core::StatsSnapshot) -> Self {
        Self {
            format: value.format.to_owned(),
            session_count: value.session_count,
            node_count: value.node_count,
            puts: value.puts,
            appends: value.extends,
            forks: value.forks,
            lookups: value.lookups,
            lookup_hits: value.lookup_hits,
            lookup_misses: value.lookup_misses,
            checksum_rejects: value.checksum_rejects,
            sessions_evicted: value.sessions_evicted,
            import_rejects: value.import_rejects,
            revision_conflicts: value.revision_conflicts,
            path_counts: value.path_counts,
        }
    }
}
use std::collections::BTreeMap;

#[cfg(test)]
mod tests {
    use super::*;

    /// Every run-time code the router can record maps onto this crate's
    /// name for it, and an unrecognised one is passed through rather
    /// than turned into a code that does not mean it.
    #[test]
    fn every_recorded_reason_maps_onto_the_frozen_namespace() {
        for (code, expected) in [
            (
                toktier_routing_core::R_INPUT_BELOW_GPU_THRESHOLD,
                ReasonCode::InputBelowGpuThreshold,
            ),
            (
                toktier_routing_core::R_INPUT_ADDED_TOKEN,
                ReasonCode::InputAddedToken,
            ),
            (
                toktier_routing_core::R_INPUT_GUARD_ROUTED,
                ReasonCode::InputGuardRouted,
            ),
            (
                toktier_routing_core::R_EXEC_FAULT,
                ReasonCode::ExecutionFault,
            ),
            (
                toktier_routing_core::R_INPUT_POSTPROCESS_ROUTED,
                ReasonCode::InputPostprocessRouted,
            ),
            (toktier_routing_core::R_CHUNK_NO_CUT, ReasonCode::ChunkNoCut),
            (
                toktier_routing_core::R_CHUNK_FAMILY_UNSUPPORTED,
                ReasonCode::ChunkFamilyUnsupported,
            ),
            (
                toktier_routing_core::R_CHUNK_ROUTE_NOT_FAST_CPU,
                ReasonCode::ChunkRouteNotFastCpu,
            ),
        ] {
            assert_eq!(ReasonCode::from_ledger_code(code), expected, "for {code}");
        }
        assert_eq!(
            ReasonCode::from_ledger_code(R_INVALID_PRIOR_STATE),
            ReasonCode::InvalidPriorState
        );
        assert_eq!(
            ReasonCode::from_ledger_code("R_SOMETHING_LATER"),
            ReasonCode::Other("R_SOMETHING_LATER".to_owned())
        );
    }
}