toktier 0.2.0

Rust API preview for TokTier's exact, stateful tokenization runtime
Documentation
//! Exact embedded release data used by Rust air-gap exports.

macro_rules! packaged {
    ($path:literal) => {
        include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/data/", $path)) as &[u8]
    };
}

pub(crate) const FILES: &[(&str, &[u8])] = &[
    ("THIRD_PARTY_NOTICES", packaged!("THIRD_PARTY_NOTICES")),
    (
        "packaging/fast_cpu/LICENSE-gigatoken",
        packaged!("packaging/fast_cpu/LICENSE-gigatoken"),
    ),
    (
        "packaging/fast_cpu/NOTICE-gigatoken-pinned",
        packaged!("packaging/fast_cpu/NOTICE-gigatoken-pinned"),
    ),
    (
        "packaging/fast_cpu/THIRD_PARTY_LICENSES-gigatoken.txt",
        packaged!("packaging/fast_cpu/THIRD_PARTY_LICENSES-gigatoken.txt"),
    ),
    (
        "packaging/fast_cpu/gigatoken-toktier-pinned-1.patch",
        packaged!("packaging/fast_cpu/gigatoken-toktier-pinned-1.patch"),
    ),
    (
        "build/source_identity.json",
        packaged!("build/source_identity.json"),
    ),
    (
        "licenses/RUST_DEPENDENCY_LICENSES.txt",
        packaged!("licenses/RUST_DEPENDENCY_LICENSES.txt"),
    ),
    (
        "sbom/toktier.cyclonedx.json",
        packaged!("sbom/toktier.cyclonedx.json"),
    ),
    (
        "schemas/evidence_manifest.schema.json",
        packaged!("schemas/evidence_manifest.schema.json"),
    ),
    (
        "schemas/sibling_alias_source.schema.json",
        packaged!("schemas/sibling_alias_source.schema.json"),
    ),
    (
        "schemas/sibling_aliases.schema.json",
        packaged!("schemas/sibling_aliases.schema.json"),
    ),
    (
        "schemas/support_registry.schema.json",
        packaged!("schemas/support_registry.schema.json"),
    ),
    (
        "src/toktier/artifacts/tables/artifact_manifest.v1.json",
        packaged!("src/toktier/artifacts/tables/artifact_manifest.v1.json"),
    ),
    (
        "src/toktier/artifacts/tables/sibling_aliases.v1.json",
        packaged!("src/toktier/artifacts/tables/sibling_aliases.v1.json"),
    ),
    (
        "src/toktier/kernels/prebuilt/build_manifest.json",
        packaged!("src/toktier/kernels/prebuilt/build_manifest.json"),
    ),
    (
        "src/toktier/kernels/prebuilt/pretok_kernel.fatbin",
        packaged!("src/toktier/kernels/prebuilt/pretok_kernel.fatbin"),
    ),
    (
        "src/toktier/kernels/prebuilt_unit.cu",
        packaged!("src/toktier/kernels/prebuilt_unit.cu"),
    ),
    (
        "src/toktier/kernels/pretok_kernel.cu",
        packaged!("src/toktier/kernels/pretok_kernel.cu"),
    ),
    (
        "src/toktier/kernels/tables/kernel_families.v1.json",
        packaged!("src/toktier/kernels/tables/kernel_families.v1.json"),
    ),
    (
        "src/toktier/kernels/tables/nfc_quick_check.v1.meta.json",
        packaged!("src/toktier/kernels/tables/nfc_quick_check.v1.meta.json"),
    ),
    (
        "src/toktier/kernels/tables/nfc_quick_check.v1.npy",
        packaged!("src/toktier/kernels/tables/nfc_quick_check.v1.npy"),
    ),
    (
        "src/toktier/kernels/tables/pretok_classes_cl100k.v3.npy",
        packaged!("src/toktier/kernels/tables/pretok_classes_cl100k.v3.npy"),
    ),
    (
        "src/toktier/kernels/tables/pretok_classes_cl100k_marks_as_letters.v3.npy",
        packaged!("src/toktier/kernels/tables/pretok_classes_cl100k_marks_as_letters.v3.npy"),
    ),
    (
        "src/toktier/kernels/tables/pretok_classes_deepseek.v1.meta.json",
        packaged!("src/toktier/kernels/tables/pretok_classes_deepseek.v1.meta.json"),
    ),
    (
        "src/toktier/kernels/tables/pretok_classes_deepseek.v1.npy",
        packaged!("src/toktier/kernels/tables/pretok_classes_deepseek.v1.npy"),
    ),
    (
        "src/toktier/kernels/tables/pretok_classes_kimi.v1.meta.json",
        packaged!("src/toktier/kernels/tables/pretok_classes_kimi.v1.meta.json"),
    ),
    (
        "src/toktier/kernels/tables/pretok_classes_kimi.v1.npy",
        packaged!("src/toktier/kernels/tables/pretok_classes_kimi.v1.npy"),
    ),
    (
        "src/toktier/kernels/tables/pretok_classes_o200k.v4.npy",
        packaged!("src/toktier/kernels/tables/pretok_classes_o200k.v4.npy"),
    ),
    (
        "src/toktier/repair/tables/fast_repair_families.v1.json",
        packaged!("src/toktier/repair/tables/fast_repair_families.v1.json"),
    ),
    (
        "src/toktier/repair/tables/repair_pclass.v1.zlib",
        packaged!("src/toktier/repair/tables/repair_pclass.v1.zlib"),
    ),
    (
        "src/toktier/routing/tables/support_registry.v1.json",
        packaged!("src/toktier/routing/tables/support_registry.v1.json"),
    ),
];