inferencelayer 0.2.4

Kortexya's engine-native inference layer — LLM generation + embedding/encoder family on wgpu (WGSL kernels, any adapter) with a pure-Rust CPU fallback
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60

//! `lfm2-shard-worker` — one pipeline-parallel stage of a sharded model (see `inferencelayer::shard`).
//!
//! ```text
//! lfm2-shard-worker [--model <dir>] [--layers <start>:<end>] [--vram-gb <g>] [--token <t>] \
//!                   [--cache-dir <dir>] --listen 0.0.0.0:9301
//! ```
//! With `--layers` (requires `--model`) the range is PINNED (loads at startup). Without it the
//! worker is AUTO: it waits for the coordinator's OP_LOAD assignment — the auto-split path,
//! where the coordinator plans ranges across the fleet (proportional to `--vram-gb` when every
//! stage reports one). Without `--model` the worker is MODELLESS: the coordinator SHIPS it
//! exactly its layers' bytes, cached under `--cache-dir` (env `OSFKB_SHARD_CACHE`, default
//! `~/.cache/lfm2-shards`). `--token` (or env `OSFKB_SHARD_TOKEN`) requires the fleet token on
//! every connection.
use anyhow::{Context, Result};

fn main() -> Result<()> {
    // Surface webrtc-rs's internal ICE/DTLS/TURN `log` output when `RUST_LOG` is set (diagnostics).
    #[cfg(feature = "webrtc")]
    let _ = env_logger::try_init();
    let mut model: Option<String> = None;
    let mut layers: Option<String> = None;
    let mut listen = "0.0.0.0:9301".to_string();
    let mut vram_gb: Option<f64> = None;
    let mut token: Option<String> = None;
    let mut cache_dir: Option<std::path::PathBuf> = None;
    let mut join: Option<String> = None;
    let mut advertise: Option<String> = None;
    let mut args = std::env::args().skip(1);
    while let Some(a) = args.next() {
        match a.as_str() {
            "--model" => model = args.next(),
            "--layers" => layers = args.next(),
            "--listen" => listen = args.next().context("--listen value")?,
            "--vram-gb" => vram_gb = Some(args.next().context("--vram-gb value")?.parse()?),
            "--token" => token = args.next(),
            "--cache-dir" => cache_dir = args.next().map(std::path::PathBuf::from),
            "--join" => join = args.next(),
            "--advertise" => advertise = args.next(),
            other => anyhow::bail!("unknown argument {other}"),
        }
    }
    let layers = layers
        .map(|l| -> Result<(usize, usize)> {
            let (s, e) = l.split_once(':').context("--layers must be start:end")?;
            Ok((s.parse()?, e.parse()?))
        })
        .transpose()?;
    inferencelayer::shard::run_worker(
        &listen,
        model.as_deref().map(std::path::Path::new),
        inferencelayer::WorkerOptions {
            layers,
            vram_gb,
            token,
            cache_dir,
            join,
            advertise,
        },
    )
}