inferencelayer 0.2.2

# Standalone, cross-vendor (Metal/Vulkan/DX12/GL) inference engine on wgpu + hand-written WGSL
# kernels, with a pure-Rust CPU fallback. LFM2/Qwen3 generation, the encoder family (BERT, XLM-R,
# ColBERT, Qwen3-embed, SigLIP, CrossEncoder), sharded fleet serving, and grammar-constrained
# decoding. Extracted from osfkb "crates/lfm2-wgpu" (history preserved); no OSFKB deps — it builds
# and benchmarks standalone. Correctness oracle = the candle LFM2 reference.
[package]
name = "inferencelayer"
version = "0.2.2"
edition = "2024"
authors = ["Kortexya"]
# Proprietary — Kortexya owner; see LICENSE. Published to crates.io as SOURCE-AVAILABLE under
# that proprietary license (crates.io does not require an open-source license; the license
# governs use, publication only makes the source readable). Version must match the release
# tag — the CI publish job enforces it.
license-file = "LICENSE"
description = "Kortexya's engine-native inference layer — LLM generation + embedding/encoder family on wgpu (WGSL kernels, any adapter) with a pure-Rust CPU fallback"
repository = "https://gitlab.com/kortexya/inferencelayer"
readme = "README.md"
# crates.io ships ONLY the buildable surface: the 100+ MB of oracle fixtures (tests/) and every
# internal artifact (bench notes, Dockerfile, CI, .claude/) stay out of the public package.
include = ["src/**", "web/moshi_webrtc_demo.html", "LICENSE", "README.md"]

[dependencies]
wgpu = "30"
pollster = "0.4"
bytemuck = { version = "1.18", features = ["derive"] }
half = { version = "2.4", features = ["bytemuck"] }
safetensors = "0.4"
serde_json = "1.0"
anyhow = "1.0"
# Tokenizer — used by the CLIs/tests only, never by the engine core (so the core stays
# wasm-targetable: `tokenizers` pulls the `onig_sys` C library that has no wasm build).
tokenizers = { version = "0.22.2", optional = true }
# The gliner2 relation processor's word splitter (its `WhitespaceTokenSplitter` is a specific regex —
# it splits "hawaii." into "hawaii"+".", which the tokenization+offsets depend on). CLI/tests only.
regex = { version = "1", optional = true }
# HTTP serving layer (optional adapter — the engine core stays dependency-light/standalone).
axum = { version = "0.8", features = ["macros", "ws"], optional = true }
tokio = { version = "1.42", features = ["full"], optional = true }
serde = { version = "1.0", features = ["derive", "rc"], optional = true }
futures-core = { version = "0.3", optional = true }
# Fleet transport (`shard`/`shard_serve`): std::net/thread, native-only.
tungstenite = { version = "0.26", optional = true }
# Mic/speaker I/O for the live full-duplex demo (`moshi-live`). Feature-gated like webrtc:
# CoreAudio/ALSA/WASAPI backends are native-only weight the default build must not carry.
cpal = { version = "0.16", optional = true }
# Opus audio codec (libopus C bindings) for the WebRTC media leg of `moshi-serve`. The one
# non-Rust dependency in the tree, gated behind `webrtc-media` and OFF by default — WebRTC
# audio is Opus-over-RTP by the browser spec, so it is unavoidable for that path (Kyutai's own
# moshi-server uses the same crate). The pure-Rust default build never links it.
opus = { version = "0.3", optional = true }

# Native WebRTC data-channel endpoint (feature `webrtc`, native-only) — the last-rung P2P
# transport for peers behind NAT. Off by default: pulls a large async stack (tokio + ICE/DTLS/SCTP),
# so only the `webrtc` build carries it. Latest stable; 0.20 is prerelease-only at time of writing.
webrtc = { version = "0.17", optional = true }
# Only for the WebRTC data-channel byte pipe (`dc.send(&Bytes)`); same version `webrtc` resolves.
bytes = { version = "1", optional = true }
# In-process TURN server for the hermetic relay-fallback proof (`spawn_local_turn_server`). Both are
# already transitive deps of `webrtc`, so naming them here is zero extra compilation; native-only.
turn = { version = "0.17", optional = true }
webrtc-util = { version = "0.17", optional = true }
# Surfaces webrtc-rs's internal `log` output (ICE/DTLS/TURN) for diagnostics; init in the CLI bins.
env_logger = { version = "0.11", optional = true }
# webrtc-rs's DTLS uses rustls 0.23, which panics unless a process-level CryptoProvider is installed;
# we install `ring` explicitly at startup. Pinning the feature here also makes the lean worker binary
# resolve rustls WITH a crypto backend (the full workspace gets one transitively; the bin doesn't).
rustls = { version = "0.23", default-features = false, features = ["ring"], optional = true }

# CPU encoder runtime + parity oracle (feature `encoder-cpu`, native-only). `gemm` is the exact
# pure-Rust backend candle-CPU uses (already compiled in the workspace via candle 0.10), so the
# CPU embedding path competes on equal footing without any candle dependency; rayon parallelizes
# the non-gemm stages; libm supplies exact erff for erf-GELU.
gemm = { version = "0.19", optional = true }
rayon = { version = "1.11", optional = true }
libm = { version = "0.2", optional = true }
# Image DECODING only (PNG/JPEG/WebP) for the vision tower — default features off, so no font,
# no filesystem codecs, and no C dependency: the resampler and every other pixel op is ours.
image = { version = "0.25", default-features = false, features = [
    "png",
    "jpeg",
    "webp",
], optional = true }
cudarc = { version = "0.19.8", default-features = false, features = ["driver", "nvrtc", "cublas", "f16", "dynamic-loading", "cuda-12020", "std"], optional = true }

# wasm-only: async GPU readback awaits the `map_async` completion (WebGPU has no blocking
# poll). Native never compiles this — the sync `read_stage_out` uses std::sync::mpsc.
[target.'cfg(target_arch = "wasm32")'.dependencies]
futures-channel = "0.3"

[features]
# The native default: the pipeline-sharding fleet layer + the CLI tokenizer. A wasm build
# (`--no-default-features`) drops both, leaving only the wgpu compute core.
default = ["net", "cli", "encoder-cpu"]
# `net` gates the std::net/thread fleet transport modules (`shard`, `shard_serve`) and their
# re-exports; the compute core (forward/deltanet/turboquant/server/weights) never needs it.
net = []
# `cli` = the tokenizer dependency for the command-line binaries.
cli = ["dep:tokenizers", "dep:regex"]
# The CPU embedding-encoder runtime (no-GPU fallback) + the parity oracle for the GPU encoder
# kernels. Native-only (a wasm build is GPU-only by construction).
encoder-cpu = ["dep:gemm", "dep:rayon", "dep:libm", "dep:image"]
# `server` implies `encoder-cpu`: the serving binary embeds via EmbedEngine::auto, whose CPU
# fallback is what keeps GPU-less hosts serving.
server = ["dep:axum", "dep:tokio", "dep:serde", "dep:futures-core", "cli", "encoder-cpu"]
# WebSocket transport for the shard fleet (join/hops/sink accept `ws://host:port` addresses;
# listeners auto-detect WS vs raw-TCP peers on one port). The browser-stage prerequisite.
ws = ["dep:tungstenite", "net"]
# Native WebRTC data-channel endpoint: a feature-gated optional adapter (per the feature-flags rule)
# so V100/Mac/Linux workers can speak data channels to browser or native peers. Implies `net` (the
# `shard` OP_SIGNAL arm lives there) and needs a tokio runtime to drive the async `webrtc-rs` stack.
webrtc = ["dep:webrtc", "dep:tokio", "dep:bytes", "dep:turn", "dep:webrtc-util", "dep:env_logger", "dep:rustls", "net"]

# Audio device I/O (mic + speaker) for the live demo binaries.
audio-io = ["dep:cpal"]
# WebRTC media leg for `moshi-serve` — SDP /v1/realtime/calls + Opus/RTP tracks + oai-events data
# channel. Pulls the webrtc-rs stack + libopus (the tree's only C dependency); OFF by default.
webrtc-media = ["server", "webrtc", "dep:opus"]
cudarc = ["dep:cudarc"]

# Dev-only: an INDEPENDENT JSON-Schema validator used as the corpus oracle for the G1 grammar
# gate (every FSM-accepted byte string must also validate here). Never linked into the engine.
[dev-dependencies]
jsonschema = "0.47"
# The GLM-OCR vision parity gate decodes the oracle's embedded PNG.
base64 = "0.22"
# Dump naga's SPIR-V for the hot GEMV to inspect codegen (compiler-wall investigation).
naga = { version = "30", features = ["wgsl-in", "spv-out"] }

[[bin]]
name = "lfm2-serve"
path = "src/bin/serve.rs"
required-features = ["server"]

[[bin]]
name = "cuda-tower-gate"
path = "src/bin/cuda_tower_gate.rs"
required-features = ["cudarc", "encoder-cpu"]

[[bin]]
name = "deepencoder-bench"
path = "src/bin/deepencoder_bench.rs"

[[bin]]
name = "lfm2-shard-worker"
path = "src/bin/shard_worker.rs"
required-features = ["net"]

[[bin]]
name = "lfm2-shard-run"
path = "src/bin/shard_run.rs"
required-features = ["net", "cli"]

[[bin]]
name = "lfm2-generate"
path = "src/bin/generate.rs"
required-features = ["cli"]

[[bin]]
name = "lfm2-profile-stage"
path = "src/bin/profile_stage.rs"
required-features = ["net"]

[[bin]]
name = "kernel-lab"
path = "src/bin/kernel_lab.rs"

# Pure-CPU Qwen3.5 (NuExtract-3) decode parity check against the transformers gold. No GPU.
[[bin]]
name = "qwen35-cpu"
path = "src/bin/qwen35_cpu.rs"

# The hold-the-crown perf ledger (core-only: engine rows + pins gate; fleet rows via --record).
[[bin]]
name = "scoreboard"
path = "src/bin/scoreboard.rs"

# The quantization RULER: teacher-forced KL divergence between a reference arm and a candidate
# arm over a calibration corpus (the metric that ranks quant schemes; see src/kld.rs).
[[bin]]
name = "kld-eval"
path = "src/bin/kld_eval.rs"
required-features = ["cli"]

# Native GGUF producer: safetensors -> quantized GGUF with in-repo encoders (no llama.cpp).
[[bin]]
name = "gguf-quantize"
path = "src/bin/gguf_quantize.rs"
required-features = ["cli"]

# Per-model precision policy search: greedy dKLD/dMB over blk:/head candidates, judged by the
# KLD ruler (the engine's own "Dynamic 2.0" tuner; see src/bin/precision_search.rs).
[[bin]]
name = "precision-search"
path = "src/bin/precision_search.rs"
required-features = ["cli"]

# The CROWN gate: engine vs PyTorch on the production encoder workloads (torch baselines ingested
# from tests/fixtures/bench_encoders.py; hold + crown gates).
[[bin]]
name = "encoder-scoreboard"
path = "src/bin/encoder_scoreboard.rs"
required-features = ["encoder-cpu", "cli"]

# Pocket TTS: text + a voice (a pre-computed KV cache) -> a 24 kHz WAV, streamed per 80 ms frame.
[[bin]]
name = "pocket-tts"
path = "src/bin/pocket_tts.rs"
required-features = ["encoder-cpu", "cli"]

# whisper speech-to-text: encoder on the GPU (Metal/Vulkan/DX/WebGPU) when available else CPU,
# OpenAI-Realtime-compatible WS server over the duplex stack (client_secrets + /v1/realtime).
[[bin]]
name = "moshi-serve"
path = "src/bin/moshi_serve.rs"
required-features = ["server"]

# Live full-duplex conversation: mic -> Moshi -> speaker, with transcript + action hooks.
[[bin]]
name = "moshi-live"
path = "src/bin/moshi_live.rs"
required-features = ["encoder-cpu", "cli", "audio-io"]

# Voice-message demo of the full-duplex stack: WAV in -> Moshi's spoken reply + transcript.
[[bin]]
name = "moshi-talk"
path = "src/bin/moshi_talk.rs"
required-features = ["encoder-cpu", "cli"]

[[bin]]
name = "moshi-converse"
path = "src/bin/moshi_converse.rs"
required-features = ["encoder-cpu", "cli"]

# KV-cached greedy decode + BPE detokenize. `whisper <model-dir> <audio.wav>` (16 kHz mono PCM16).
[[bin]]
name = "whisper"
path = "src/bin/whisper.rs"
required-features = ["encoder-cpu", "cli"]

# Chatterbox Phase-0 manifest dumper: parse each checkpoint's safetensors header (Rust-only).
[[bin]]
name = "chatterbox-dbg"
path = "src/bin/chatterbox_dbg.rs"
required-features = ["encoder-cpu", "cli"]

# Chatterbox provenance checker: does this WAV carry our clone watermark? (exit 0 = yes)
[[bin]]
name = "chatterbox-wmcheck"
path = "src/bin/chatterbox_wmcheck.rs"
required-features = ["encoder-cpu", "cli"]

# Qwen3-TTS Phase-0 manifest dumper + config validator (Rust-only; see HANDOFF_qwen3tts.md).
[[bin]]
name = "qwen3tts-dbg"
path = "src/bin/qwen3tts_dbg.rs"
required-features = ["encoder-cpu", "cli"]

# Backbone-conv A/B arm for the Docling layout port (see bench/conv_ab.py).
[[bin]]
name = "conv-bench"
path = "src/bin/conv_bench.rs"
required-features = ["encoder-cpu"]

# Layout-model A/B arm (torch twin: tests/fixtures/bench_layout_torch.py).
[[bin]]
name = "layout-bench"
path = "src/bin/layout_bench.rs"
required-features = ["encoder-cpu"]

# TableFormer manual A/B: page/crop PNG → OTSL tags + cell boxes (JSON).
[[bin]]
name = "tableformer"
path = "src/bin/tableformer.rs"
required-features = ["encoder-cpu"]