furnace-server 0.14.0

Fast, unified, Rust-native, Axum-compatible inference endpoints and nervous system for tensor models and LLMs: KServe / Triton and OpenAI APIs from one pure-Rust server
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60

//! Scaffolding for the nerve's unit tests

use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};

use super::fabric::Fabric;
use super::neuron::{Firing, FnNeuron, Neuron, NeuronKind};
use super::receptor::Receptor;
use super::signal::{Feature, NeuronId, Payload, Signal};

/// Wait for `f` to hold, checking every millisecond; fails the test after 10 seconds
pub(crate) fn wait_until(f: impl Fn() -> bool) {
    let deadline = Instant::now() + Duration::from_secs(10);
    while !f() {
        assert!(Instant::now() < deadline, "still not true after 10 s");
        std::thread::sleep(Duration::from_millis(1));
    }
}

/// A receptor that passes everything on
pub(crate) fn pass(name: &str) -> impl Neuron {
    FnNeuron::reflex(name, |s, _| Firing::one(s)).with_kind(NeuronKind::Receptor)
}

/// A fabric where a receptor `in` reaches `neuron` through a reflex; stimulate the id returned
pub(crate) fn fabric_with(neuron: impl Neuron) -> (Fabric, NeuronId) {
    let mut b = Fabric::builder();
    let r = b.neuron(pass("in"));
    let n = b.neuron(neuron);
    b.reflex(r, n);
    (b.build().unwrap(), r)
}

/// A started fabric with `receptor` and a sink behind it that records what arrives
pub(crate) fn fabric_with_receptor<R: Receptor>(
    receptor: R,
) -> (Fabric, NeuronId, Arc<Mutex<Vec<Signal>>>) {
    let seen = Arc::new(Mutex::new(Vec::new()));
    let mut b = Fabric::builder();
    let r = b.receptor(receptor);
    let sink = {
        let seen = seen.clone();
        b.neuron(
            FnNeuron::reflex("sink", move |s, _| {
                seen.lock().unwrap().push(s);
                Firing::Quiet
            })
            .with_kind(NeuronKind::Effector),
        )
    };
    b.reflex(r, sink);
    let fabric = b.build().unwrap();
    fabric.start();
    (fabric, r, seen)
}

/// A command for the effector named `effector`
pub(crate) fn command(effector: &str, args: serde_json::Value) -> Signal {
    Signal::new(Feature::NONE, Payload::None).command(effector, args)
}