cortiq_engine/
lib.rs

1//! Cortiq inference engine — sparse forward pass, attention, tokenization, sampling.
2
3pub mod attention;
4pub mod chat_template;
5pub mod cpuprof;
6pub mod audiovae;
7pub mod blr;
8pub mod bounded;
9pub mod d1;
10pub mod d1_parse;
11pub mod dflash;
12pub mod dit;
13pub mod dsv4;
14pub mod dory;
15pub mod dsv41;
16pub mod dsv41_encoding;
17pub mod dsv41_vision;
18pub mod egemma2;
19pub mod egemma2_audio;
20pub mod egemma2_mm;
21pub mod egemma2_vision;
22pub(crate) mod expert_store;
23pub mod fcd;
24pub mod fcd_ops;
25#[cfg(feature = "gpu")]
26pub mod nvg_train;
27pub mod f32_backend;
28pub mod g3n;
29pub mod g4e;
30pub mod gptq_capture;
31pub mod gpu;
32#[cfg(target_os = "macos")]
33pub mod gpu_metal;
34#[cfg(feature = "gpu")]
35pub mod gpu_wgpu;
36/// The wgpu crate the device types of `gpu_wgpu` come from (offline tools
37/// hold buffers across calls).
38#[cfg(feature = "gpu")]
39pub use wgpu;
40pub mod imagegen;
41pub mod inference;
42pub mod kv_cache;
43pub mod lfm3_audio;
44pub mod lfm3_speech;
45pub mod lfm3_vl;
46pub mod linear_core;
47pub mod loader;
48pub mod lookup;
49pub mod mimo_moe;
50pub(crate) mod minicpm_tail;
51pub mod ltxaudio;
52pub mod ltxdit;
53pub mod ltxdur;
54pub mod ltxenc;
55pub mod ltxlora;
56pub mod ltxpipe;
57pub mod ltxte;
58pub mod ltxups;
59pub mod ltxvae;
60pub mod media;
61pub mod mimo_audio;
62pub mod mimo_mm;
63pub mod mimo_ingress;
64pub mod mimo_vision;
65pub mod minicpm_ocr;
66pub mod onevision2;
67pub mod mm_ab;
68pub mod mmh3;
69pub mod mmh3ups;
70pub mod music3;
71pub mod moss_image;
72pub mod moss_imagegen;
73pub mod nystrom;
74pub mod pin;
75pub mod pipeline;
76/// Gradium Phonon text-to-speech execution for self-contained CMF packages.
77#[cfg(feature = "phonon")]
78pub mod phonon;
79pub mod pool;
80pub mod primal_vae;
81pub mod prism;
82pub mod qtensor;
83pub mod qwen3te;
84pub mod qwen3vis;
85pub mod qwen4_exp;
86pub mod qwen4_o1;
87pub mod qwen_image;
88pub mod qwen_image21;
89pub mod qwen_image21_vae;
90pub mod qwen_image21gen;
91pub mod qwen_image_encoder;
92pub mod qwen_image_ops;
93pub mod qwen_image_vae;
94pub mod qwen_image_vision;
95pub mod qwen_imagegen;
96pub mod router;
97pub mod runtime;
98pub mod s2v;
99pub mod s2v_enc;
100pub mod s2v_vae;
101pub mod s2vgen;
102pub mod sam3;
103pub mod sam3vid;
104pub(crate) mod sam3vid_net;
105pub mod sampler;
106pub mod sensenova;
107pub mod skillbake;
108pub mod swarm;
109pub mod textenc;
110pub mod tokenizer;
111pub mod uocr;
112pub mod vae;
113pub mod vae3d;
114pub mod videogen;
115/// NVIDIA Nemotron 3.5 Transcribe: audio perception + transcription loop.
116pub mod nemotron_asr;
117/// Nemotron-H hybrid Mamba-2 / attention / MoE decoder.
118pub mod nemotron_h;
119/// Native Whisper encoder/decoder inference over CMF checkpoints.
120pub mod whisper;
121pub mod zimage;
122pub mod zimagegen;
123/// The native Vulkan lane — an accelerator behind a capability probe,
124/// present only where Vulkan is.
125#[cfg(all(
126    feature = "gpu",
127    any(target_os = "linux", target_os = "windows", target_os = "android")
128))]
129pub use nystrom::NystromState;
130pub use pipeline::{GenerateResult, MtpFfnProbe, Pipeline, TokenCallback, TokenTrace};
131pub use runtime::CortiqRuntime;
132
133/// Test-only: N empty Metal command-buffer round trips, total seconds.
134#[doc(hidden)]
135#[cfg(target_os = "macos")]
136pub fn gpu_empty_submit_for_test(n: usize) -> f64 {
137    gpu_metal::empty_submit_bench(n)
138}
139
140/// Test-only: N pipelined empty submits, one final wait.
141#[doc(hidden)]
142#[cfg(target_os = "macos")]
143pub fn gpu_pipelined_submit_for_test(n: usize) -> f64 {
144    gpu_metal::pipelined_submit_bench(n)
145}
146
147/// Test-only: build a q1 MoeJob trio (weight 1.0).
148#[doc(hidden)]
149#[cfg(target_os = "macos")]
150pub fn gpu_moe_job_for_test(
151    gi: usize,
152    ui: usize,
153    di: usize,
154    inter: usize,
155    hidden: usize,
156    x: Vec<f32>,
157) -> gpu::MoeJob<'static> {
158    gpu::MoeJob {
159        gate: (gi, inter, hidden, &[]),
160        up: (ui, inter, hidden, &[]),
161        down: (di, hidden, inter, &[]),
162        xs_gate: x.clone(),
163        xs_up: x,
164        down_col: &[],
165        w: 1.0,
166        q1: true,
167        q4t: false,
168        q4tp: false,
169        gu_q2: false,
170        swiglu_limit: 0.0,
171        act: 0,
172    }
173}
174
175/// Test-only: run the metal moe_block on one job.
176#[doc(hidden)]
177#[cfg(target_os = "macos")]
178pub fn gpu_moe_block_for_test(
179    model: &std::sync::Arc<cortiq_core::CmfModel>,
180    job: gpu::MoeJob<'_>,
181    out: &mut [f32],
182) -> bool {
183    gpu_metal::moe_block(model, &[job], out)
184}
185
186/// Test-only: q1 matvec_batch — jobs (idx, rows, cols); first two share
187/// x, the third takes xi.
188#[doc(hidden)]
189#[cfg(target_os = "macos")]
190pub fn gpu_batch_q1_for_test(
191    model: &std::sync::Arc<cortiq_core::CmfModel>,
192    shapes: &[(usize, usize, usize)],
193    x: &[f32],
194    xi: &[f32],
195    outs: &mut [&mut [f32]],
196) -> bool {
197    let jobs: Vec<gpu::BatchJob> = shapes
198        .iter()
199        .enumerate()
200        .map(|(k, &(idx, rows, cols))| gpu::BatchJob {
201            idx,
202            rows,
203            cols,
204            row_scale: &[],
205            xs: if k < 2 { x.to_vec() } else { xi.to_vec() },
206            layout: gpu::BatchLayout::Q1,
207        })
208        .collect();
209    gpu_metal::matvec_batch(model, &jobs, outs)
210}
211
212/// Test-only direct handle to the Metal q1 matvec (micro-benchmarks).
213#[doc(hidden)]
214#[cfg(target_os = "macos")]
215pub fn gpu_q1_matvec_for_test(
216    model: &std::sync::Arc<cortiq_core::CmfModel>,
217    idx: usize,
218    xs: &[f32],
219    rows: usize,
220    cols: usize,
221    out: &mut [f32],
222) -> bool {
223    gpu_metal::q1_matvec(model, idx, xs, rows, cols, out)
224}
225pub use sampler::SamplerConfig;
226/// Re-exported for offline tools (the NVG encoder's f64 GEMMs in the CLI).
227pub use matrixmultiply;
228
229/// Device forward of one S2V input for `cortiq s2v-check` (None without a device path).
230pub fn s2v_check_device(dit: &s2v::S2vDit, i: &s2v::S2vInput<'_>) -> Option<Vec<f32>> {
231    gpu::s2v_check_forward(dit, i)
232}
cortiq_engine/lib.rs

cortiq_engine/
lib.rs
