Struct LlmExecutorFactory

Source

pub struct LlmExecutorFactory;

Expand description

Candle executor factory Builds a ModelExecutor from a resolved model path. Despite the historical “candle” name (PR #127 deleted the legacy CandleBackend trait), this factory now produces real Backend-based executors for LLMs (LlamaFamilyModel / Qwen3MoeModel) and candle-based executors for embedding / multimodal / ASR (Bert, CLIP, Whisper).

The factory dispatches over four axes:

Dim 1 (architecture): LlamaFamilyModel vs Qwen3MoeModel — runtime value, decided inside [build_llm].
Dim 3 (weight format): safetensors vs GGUF — runtime branch on WeightFormat::detect() at the top of Self::create.
Dim 4 (device): CpuBackend / MetalBackend / CudaBackend — type parameter B, picked by the (device, kv_dtype) cascade below.
Dim 5 (kv dtype): KvFp16 / KvInt8 / KvFp8 — type parameter K, same cascade. Only KvFp16 is wired today; KvInt8 lands in PR C (see docs/dim5-model-wireup-plan.md).

LlmExecutorFactory

Struct LlmExecutorFactory Copy item path

Trait Implementations§

impl ComponentFactory<Arc<dyn ModelExecutor + Send + Sync>> for LlmExecutorFactory

fn create<'life0, 'life1, 'async_trait>( &'life0 self, config: &'life1 ComponentConfig, ) -> Pin<Box<dyn Future<Output = Result<Arc<dyn ModelExecutor + Send + Sync>>> + Send + 'async_trait>>where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

fn metadata(&self) -> ComponentMetadata

Auto Trait Implementations§

impl Freeze for LlmExecutorFactory

impl RefUnwindSafe for LlmExecutorFactory

impl Send for LlmExecutorFactory

impl Sync for LlmExecutorFactory

impl Unpin for LlmExecutorFactory

impl UnsafeUnpin for LlmExecutorFactory

impl UnwindSafe for LlmExecutorFactory

Blanket Implementations§

impl<T> Any for Twhere T: 'static + ?Sized,

fn type_id(&self) -> TypeId

impl<T> Borrow<T> for Twhere T: ?Sized,

fn borrow(&self) -> &T

impl<T> BorrowMut<T> for Twhere T: ?Sized,

fn borrow_mut(&mut self) -> &mut T

impl<T> ErasedDestructor for Twhere T: 'static,

impl<T> From<T> for T

fn from(t: T) -> T

impl<T> Instrument for T

fn instrument(self, span: Span) -> Instrumented<Self> ⓘ

fn in_current_span(self) -> Instrumented<Self> ⓘ

impl<T, U> Into<U> for Twhere U: From<T>,

fn into(self) -> U

impl<T> IntoEither for T

fn into_either(self, into_left: bool) -> Either<Self, Self> ⓘ

fn into_either_with<F>(self, into_left: F) -> Either<Self, Self> ⓘwhere F: FnOnce(&Self) -> bool,

impl<F, T> IntoSample<T> for Fwhere T: FromSample<F>,

fn into_sample(self) -> T

impl<T> Pointable for T

const ALIGN: usize

type Init = T

unsafe fn init(init: <T as Pointable>::Init) -> usize

unsafe fn deref<'a>(ptr: usize) -> &'a T

unsafe fn deref_mut<'a>(ptr: usize) -> &'a mut T

unsafe fn drop(ptr: usize)

impl<T> PolicyExt for Twhere T: ?Sized,

fn and<P, B, E>(self, other: P) -> And<T, P>where T: Sized + Policy<B, E>, P: Policy<B, E>,

fn or<P, B, E>(self, other: P) -> Or<T, P>where T: Sized + Policy<B, E>, P: Policy<B, E>,

impl<T> Same for T

type Output = T

impl<T, U> TryFrom<U> for Twhere U: Into<T>,

type Error = !

fn try_from(value: U) -> Result<T, <T as TryFrom<U>>::Error>

impl<T, U> TryInto<U> for Twhere U: TryFrom<T>,

type Error = <U as TryFrom<T>>::Error

fn try_into(self) -> Result<U, <U as TryFrom<T>>::Error>

impl<V, T> VZip<V> for Twhere V: MultiLane<T>,

fn vzip(self) -> V

impl<T> WithSubscriber for T

fn with_subscriber<S>(self, subscriber: S) -> WithDispatch<Self> ⓘwhere S: Into<Dispatch>,

fn with_current_subscriber(self) -> WithDispatch<Self> ⓘ

Struct LlmExecutorFactory

fn create<'life0, 'life1, 'async_trait>( &'life0 self, config: &'life1 ComponentConfig, ) -> Pin<Box<dyn Future<Output = Result<Arc<dyn ModelExecutor + Send + Sync>>> + Send + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

impl<T> Any for T
where T: 'static + ?Sized,

impl<T> Borrow<T> for T
where T: ?Sized,

impl<T> BorrowMut<T> for T
where T: ?Sized,

impl<T> ErasedDestructor for T
where T: 'static,

impl<T, U> Into<U> for T
where U: From<T>,

fn into_either_with<F>(self, into_left: F) -> Either<Self, Self> ⓘ
where F: FnOnce(&Self) -> bool,

impl<F, T> IntoSample<T> for F
where T: FromSample<F>,

impl<T> PolicyExt for T
where T: ?Sized,

fn and<P, B, E>(self, other: P) -> And<T, P>
where T: Sized + Policy<B, E>, P: Policy<B, E>,

fn or<P, B, E>(self, other: P) -> Or<T, P>
where T: Sized + Policy<B, E>, P: Policy<B, E>,

impl<T, U> TryFrom<U> for T
where U: Into<T>,

impl<T, U> TryInto<U> for T
where U: TryFrom<T>,

impl<V, T> VZip<V> for T
where V: MultiLane<T>,

fn with_subscriber<S>(self, subscriber: S) -> WithDispatch<Self> ⓘ
where S: Into<Dispatch>,