Skip to main content

khive_runtime/
embedder_registry.rs

1//! EmbedderRegistry — pack-extensible embedding provider surface.
2//!
3//! Packs implement [`EmbedderProvider`] and register custom models via
4//! [`crate::KhiveRuntime::register_embedder`]. Built-in lattice models are pre-registered
5//! during runtime construction and require no opt-in.
6
7use std::collections::HashMap;
8use std::sync::atomic::{AtomicBool, AtomicU8, AtomicUsize, Ordering};
9use std::sync::{Arc, Mutex, OnceLock};
10
11use async_trait::async_trait;
12use lattice_embed::{
13    CachedEmbeddingService, EmbeddingModel, EmbeddingRole, EmbeddingService,
14    NativeEmbeddingService, TokenCount, DEFAULT_MAX_BATCH_SIZE, MAX_TEXT_BYTES,
15};
16use tokio::sync::{mpsc, Notify, OnceCell};
17
18use crate::error::{RuntimeError, RuntimeResult};
19
20const ADMISSION_WAITING: u8 = 1;
21const ADMISSION_ACCEPTED: u8 = 2;
22
23struct EmbeddingAdmission {
24    deadline: khive_storage::RequestReadDeadline,
25    request: khive_storage::RequestReadContext,
26    state: AtomicU8,
27    changed: Notify,
28}
29
30impl EmbeddingAdmission {
31    fn expired(&self) -> bool {
32        tokio::time::Instant::now() >= self.deadline.async_at()
33            || self.request.stop_reason().is_some()
34    }
35}
36
37tokio::task_local! {
38    static EMBEDDING_ADMISSION: Arc<EmbeddingAdmission>;
39}
40
41/// Bound only the built-in worker's pre-admission wait, above the lattice error
42/// boundary. Providers that never enter that wait retain their own semantics.
43pub(crate) async fn with_embedding_admission<T>(
44    future: impl std::future::Future<Output = lattice_embed::Result<T>>,
45) -> RuntimeResult<T> {
46    let deadline = khive_storage::effective_request_read_deadline(
47        khive_storage::RequestReadDeadline::after(khive_storage::request_read_timeout_from_env()),
48    );
49    let timeout = deadline
50        .async_at()
51        .saturating_duration_since(tokio::time::Instant::now());
52    let admission = Arc::new(EmbeddingAdmission {
53        deadline,
54        request: khive_storage::capture_request_read_context(),
55        state: AtomicU8::new(0),
56        changed: Notify::new(),
57    });
58    EMBEDDING_ADMISSION
59        .scope(Arc::clone(&admission), async move {
60            tokio::pin!(future);
61            let stopped = async {
62                tokio::select! {
63                    _ = tokio::time::sleep_until(deadline.async_at()) => {},
64                    _ = admission.request.clone().wait_for_stop() => {},
65                }
66            };
67            tokio::pin!(stopped);
68            let mut bound_expired = false;
69            loop {
70                tokio::select! {
71                    // A ready cache hit precedes admission, including an expired bound.
72                    biased;
73                    result = &mut future => return result.map_err(RuntimeError::from),
74                    _ = admission.changed.notified() => {},
75                    _ = &mut stopped, if !bound_expired => bound_expired = true,
76                }
77                match admission.state.load(Ordering::Acquire) {
78                    // An accepted job can be followed by another admission wait.
79                    // Keep supervising that next phase without cancelling this one.
80                    ADMISSION_ACCEPTED => {}
81                    ADMISSION_WAITING if bound_expired => {
82                        return Err(RuntimeError::Storage(
83                            khive_storage::StorageError::AdmissionTimeout {
84                                operation: "embedding admission".into(),
85                                timeout_ms: u64::try_from(timeout.as_millis()).unwrap_or(u64::MAX),
86                                pool_identity: None,
87                            },
88                        ));
89                    }
90                    // A provider may enter the owned adapter later. Keep the
91                    // original expired bound without timing out its other work.
92                    _ => {}
93                }
94            }
95        })
96        .await
97}
98
99#[derive(Clone, Copy)]
100enum EmbeddingCall {
101    Generic,
102    Query,
103    Passage,
104    CountTokens(EmbeddingRole),
105}
106
107enum EmbeddingOutput {
108    Vectors(Vec<Vec<f32>>),
109    TokenCounts(Vec<TokenCount>),
110}
111
112const EMBEDDING_QUEUE_CAPACITY: usize = 32;
113const EMBEDDING_MAX_JOB_BYTES: usize = DEFAULT_MAX_BATCH_SIZE * MAX_TEXT_BYTES;
114// 32 queue slots × the normal 128-text batch × 32 KiB/text = 128 MiB in flight.
115const EMBEDDING_QUEUE_BYTE_BUDGET: usize = EMBEDDING_QUEUE_CAPACITY * 128 * MAX_TEXT_BYTES;
116
117struct InFlightBytes {
118    counter: Arc<AtomicUsize>,
119    bytes: usize,
120}
121
122impl InFlightBytes {
123    fn reserve(
124        counter: Arc<AtomicUsize>,
125        byte_budget: usize,
126        bytes: usize,
127    ) -> lattice_embed::Result<Self> {
128        let mut current = counter.load(Ordering::Acquire);
129        loop {
130            let Some(next) = current.checked_add(bytes) else {
131                return Err(lattice_embed::EmbedError::Internal(format!(
132                    "embedding worker byte budget exceeded: in-flight byte count overflowed the {byte_budget}-byte budget"
133                )));
134            };
135            if next > byte_budget {
136                return Err(lattice_embed::EmbedError::Internal(format!(
137                    "embedding worker byte budget exceeded: {current} in flight + {bytes} job bytes > {byte_budget}"
138                )));
139            }
140            match counter.compare_exchange_weak(current, next, Ordering::AcqRel, Ordering::Acquire)
141            {
142                Ok(_) => return Ok(Self { counter, bytes }),
143                Err(observed) => current = observed,
144            }
145        }
146    }
147}
148
149impl Drop for InFlightBytes {
150    fn drop(&mut self) {
151        let previous = self.counter.fetch_sub(self.bytes, Ordering::AcqRel);
152        debug_assert!(previous >= self.bytes, "embedding byte counter underflow");
153    }
154}
155
156struct EmbeddingJob {
157    texts: Vec<String>,
158    model: EmbeddingModel,
159    call: EmbeddingCall,
160    reply: tokio::sync::oneshot::Sender<lattice_embed::Result<EmbeddingOutput>>,
161    _in_flight: InFlightBytes,
162}
163
164/// Bounds non-cancellable native work to one worker and a fixed queue.
165/// Callers can detach safely because closed queued jobs are skipped before execution.
166pub(crate) struct BlockingEmbeddingService<S> {
167    inner: Arc<S>,
168    worker: OnceLock<Result<mpsc::Sender<EmbeddingJob>, String>>,
169    in_flight_bytes: Arc<AtomicUsize>,
170    byte_budget: usize,
171}
172
173impl<S> BlockingEmbeddingService<S> {
174    pub(crate) fn new(inner: Arc<S>) -> Self {
175        Self {
176            inner,
177            worker: OnceLock::new(),
178            in_flight_bytes: Arc::new(AtomicUsize::new(0)),
179            byte_budget: EMBEDDING_QUEUE_BYTE_BUDGET,
180        }
181    }
182
183    #[cfg(test)]
184    fn with_byte_budget(inner: Arc<S>, byte_budget: usize) -> Self {
185        Self {
186            inner,
187            worker: OnceLock::new(),
188            in_flight_bytes: Arc::new(AtomicUsize::new(0)),
189            byte_budget,
190        }
191    }
192}
193
194impl<S: EmbeddingService + 'static> BlockingEmbeddingService<S> {
195    fn input_bytes(texts: &[String]) -> lattice_embed::Result<usize> {
196        if texts.is_empty() {
197            return Err(lattice_embed::EmbedError::InvalidInput(
198                "no texts provided".to_owned(),
199            ));
200        }
201        let input_bytes = texts.iter().try_fold(0usize, |total, text| {
202            total.checked_add(text.len()).ok_or_else(|| {
203                lattice_embed::EmbedError::InvalidInput(format!(
204                    "embedding job input exceeds the {EMBEDDING_MAX_JOB_BYTES}-byte maximum"
205                ))
206            })
207        })?;
208        if input_bytes > EMBEDDING_MAX_JOB_BYTES {
209            return Err(lattice_embed::EmbedError::InvalidInput(format!(
210                "embedding job input is {input_bytes} bytes; maximum is {EMBEDDING_MAX_JOB_BYTES} bytes"
211            )));
212        }
213        if texts.len() > DEFAULT_MAX_BATCH_SIZE {
214            return Err(lattice_embed::EmbedError::InvalidInput(format!(
215                "batch size {} exceeds maximum {DEFAULT_MAX_BATCH_SIZE}",
216                texts.len()
217            )));
218        }
219        if let Some(text) = texts.iter().find(|text| text.len() > MAX_TEXT_BYTES) {
220            return Err(lattice_embed::EmbedError::TextTooLong {
221                length: text.len(),
222                max: MAX_TEXT_BYTES,
223            });
224        }
225        Ok(input_bytes)
226    }
227
228    fn worker(&self) -> lattice_embed::Result<&mpsc::Sender<EmbeddingJob>> {
229        self.worker
230            .get_or_init(|| {
231                let (sender, receiver) = mpsc::channel(EMBEDDING_QUEUE_CAPACITY);
232                let inner = Arc::clone(&self.inner);
233                let runtime = tokio::runtime::Handle::current();
234                std::thread::Builder::new()
235                    .name("khive-embedding".to_owned())
236                    .spawn(move || Self::run_worker(inner, runtime, receiver))
237                    .map(|_| sender)
238                    .map_err(|error| error.to_string())
239            })
240            .as_ref()
241            .map_err(|error| lattice_embed::EmbedError::Internal(error.clone()))
242    }
243
244    fn run_worker(
245        inner: Arc<S>,
246        runtime: tokio::runtime::Handle,
247        mut receiver: mpsc::Receiver<EmbeddingJob>,
248    ) {
249        while let Some(job) = receiver.blocking_recv() {
250            if job.reply.is_closed() {
251                continue;
252            }
253            let result = runtime.block_on(async {
254                match job.call {
255                    EmbeddingCall::Generic => inner
256                        .embed(&job.texts, job.model)
257                        .await
258                        .map(EmbeddingOutput::Vectors),
259                    EmbeddingCall::Query => inner
260                        .embed_query(&job.texts, job.model)
261                        .await
262                        .map(EmbeddingOutput::Vectors),
263                    EmbeddingCall::Passage => inner
264                        .embed_passage(&job.texts, job.model)
265                        .await
266                        .map(EmbeddingOutput::Vectors),
267                    EmbeddingCall::CountTokens(role) => inner
268                        .count_tokens(&job.texts, job.model, role)
269                        .await
270                        .map(EmbeddingOutput::TokenCounts),
271                }
272            });
273            let _ = job.reply.send(result);
274        }
275    }
276
277    async fn run(
278        &self,
279        texts: &[String],
280        model: EmbeddingModel,
281        call: EmbeddingCall,
282    ) -> lattice_embed::Result<Vec<Vec<f32>>> {
283        match self.run_job(texts, model, call).await? {
284            EmbeddingOutput::Vectors(vectors) => Ok(vectors),
285            EmbeddingOutput::TokenCounts(_) => Err(lattice_embed::EmbedError::Internal(
286                "embedding worker returned token counts for a vector request".to_owned(),
287            )),
288        }
289    }
290
291    async fn run_job(
292        &self,
293        texts: &[String],
294        model: EmbeddingModel,
295        call: EmbeddingCall,
296    ) -> lattice_embed::Result<EmbeddingOutput> {
297        let input_bytes = Self::input_bytes(texts)?;
298        let sender = self.worker()?;
299        let admission = EMBEDDING_ADMISSION.try_with(Arc::clone).ok();
300        let permit = if let Some(admission) = &admission {
301            admission.state.store(ADMISSION_WAITING, Ordering::Release);
302            admission.changed.notify_one();
303            if admission.expired() {
304                return std::future::pending().await;
305            }
306            sender.reserve().await.map_err(|_| {
307                lattice_embed::EmbedError::Internal(
308                    "embedding worker channel is disconnected".to_owned(),
309                )
310            })?
311        } else {
312            // Direct trait callers have no RuntimeResult boundary at which to
313            // report a typed admission timeout, so retain finite fail-fast admission.
314            sender.try_reserve().map_err(|error| match error {
315                mpsc::error::TrySendError::Full(_) => {
316                    lattice_embed::EmbedError::Internal("embedding worker queue is full".to_owned())
317                }
318                mpsc::error::TrySendError::Closed(_) => lattice_embed::EmbedError::Internal(
319                    "embedding worker channel is disconnected".to_owned(),
320                ),
321            })?
322        };
323        // Capacity and expiry can become ready in one poll. Do not publish a
324        // job after the original bound, even when reserve() returned a permit.
325        if admission
326            .as_ref()
327            .is_some_and(|admission| admission.expired())
328        {
329            return std::future::pending().await;
330        }
331        let in_flight = InFlightBytes::reserve(
332            Arc::clone(&self.in_flight_bytes),
333            self.byte_budget,
334            input_bytes,
335        )?;
336        let (reply, receiver) = tokio::sync::oneshot::channel();
337        let job = EmbeddingJob {
338            texts: texts.to_vec(),
339            model,
340            call,
341            reply,
342            _in_flight: in_flight,
343        };
344        if let Some(admission) = &admission {
345            if admission.expired() {
346                return std::future::pending().await;
347            }
348            admission.state.store(ADMISSION_ACCEPTED, Ordering::Release);
349            admission.changed.notify_one();
350        }
351        permit.send(job);
352        receiver
353            .await
354            .map_err(|error| lattice_embed::EmbedError::Internal(error.to_string()))?
355    }
356}
357
358#[async_trait]
359impl<S: EmbeddingService + 'static> EmbeddingService for BlockingEmbeddingService<S> {
360    async fn embed(
361        &self,
362        texts: &[String],
363        model: EmbeddingModel,
364    ) -> lattice_embed::Result<Vec<Vec<f32>>> {
365        self.run(texts, model, EmbeddingCall::Generic).await
366    }
367
368    async fn embed_with_role(
369        &self,
370        texts: &[String],
371        model: EmbeddingModel,
372        role: EmbeddingRole,
373    ) -> lattice_embed::Result<Vec<Vec<f32>>> {
374        // The outer cache delegates raw caller text here; preparation belongs
375        // to the native service, after the caller-input admission checks.
376        let call = match role {
377            EmbeddingRole::Generic => EmbeddingCall::Generic,
378            EmbeddingRole::Query => EmbeddingCall::Query,
379            EmbeddingRole::Passage => EmbeddingCall::Passage,
380            _ => {
381                return Err(lattice_embed::EmbedError::InvalidInput(
382                    "unsupported embedding role".to_owned(),
383                ))
384            }
385        };
386        self.run(texts, model, call).await
387    }
388
389    async fn embed_query(
390        &self,
391        texts: &[String],
392        model: EmbeddingModel,
393    ) -> lattice_embed::Result<Vec<Vec<f32>>> {
394        self.run(texts, model, EmbeddingCall::Query).await
395    }
396
397    async fn embed_passage(
398        &self,
399        texts: &[String],
400        model: EmbeddingModel,
401    ) -> lattice_embed::Result<Vec<Vec<f32>>> {
402        self.run(texts, model, EmbeddingCall::Passage).await
403    }
404
405    async fn count_tokens(
406        &self,
407        texts: &[String],
408        model: EmbeddingModel,
409        role: EmbeddingRole,
410    ) -> lattice_embed::Result<Vec<TokenCount>> {
411        match self
412            .run_job(texts, model, EmbeddingCall::CountTokens(role))
413            .await?
414        {
415            EmbeddingOutput::TokenCounts(counts) => Ok(counts),
416            EmbeddingOutput::Vectors(_) => Err(lattice_embed::EmbedError::Internal(
417                "embedding worker returned vectors for a token-count request".to_owned(),
418            )),
419        }
420    }
421
422    fn model_config(&self, model: EmbeddingModel) -> lattice_embed::ModelConfig {
423        self.inner.model_config(model)
424    }
425
426    fn supports_model(&self, model: EmbeddingModel) -> bool {
427        self.inner.supports_model(model)
428    }
429
430    fn name(&self) -> &'static str {
431        self.inner.name()
432    }
433}
434
435/// A source that can produce an [`EmbeddingService`] by name.
436///
437/// Packs implement this trait to register custom embedding backends.
438/// The runtime calls [`build`](EmbedderProvider::build) lazily — once per
439/// process per model — and caches the result. Subsequent calls to
440/// `KhiveRuntime::embedder(name)` are cheap.
441///
442/// Built-in lattice models are registered automatically via
443/// [`LatticeEmbedderProvider`]; packs need not re-register them.
444#[async_trait]
445pub trait EmbedderProvider: Send + Sync {
446    /// Stable, case-sensitive name for this embedder.
447    ///
448    /// Must be unique across all registered providers. The name is used as
449    /// the key in `KhiveRuntime::embedder(name)` lookups and as the storage
450    /// table suffix for vector indices. Use the model's canonical short form
451    /// (e.g. `"all-minilm-l6-v2"`, `"my-custom-encoder"`).
452    fn name(&self) -> &str;
453
454    /// Output vector dimension for this embedder.
455    ///
456    /// Must be consistent with what [`build`](Self::build) produces.
457    /// The runtime uses this to pre-register the vector store columns.
458    fn dimensions(&self) -> usize;
459
460    /// Construct the underlying [`EmbeddingService`].
461    ///
462    /// Called at most once per process. The result is cached in a
463    /// [`OnceCell`]; concurrent callers block on the first call and share
464    /// the result thereafter.
465    async fn build(&self) -> RuntimeResult<Arc<dyn EmbeddingService>>;
466}
467
468/// An entry in the [`EmbedderRegistry`] combining a provider with its
469/// lazy-initialized service.
470pub(crate) struct EmbedderEntry {
471    provider: Arc<dyn EmbedderProvider>,
472    cell: Arc<OnceCell<Arc<dyn EmbeddingService>>>,
473    serving: Arc<AtomicBool>,
474    /// Only the runtime's built-in lattice provider has an audited document
475    /// preparation path. Pack replacements, even under a built-in name, do not.
476    audited_document_preparation: bool,
477}
478
479impl Clone for EmbedderEntry {
480    fn clone(&self) -> Self {
481        Self {
482            provider: Arc::clone(&self.provider),
483            cell: Arc::clone(&self.cell),
484            serving: Arc::clone(&self.serving),
485            audited_document_preparation: self.audited_document_preparation,
486        }
487    }
488}
489
490/// Registry of named [`EmbedderProvider`] instances.
491///
492/// Built during `KhiveRuntime` construction and optionally extended by packs
493/// via [`crate::KhiveRuntime::register_embedder`]. The registry is internally
494/// reference-counted so `KhiveRuntime::clone()` shares the same providers
495/// and cached service instances.
496#[derive(Clone, Default)]
497pub struct EmbedderRegistry {
498    entries: HashMap<String, EmbedderEntry>,
499    ordered_names: Vec<String>,
500    bound_engines: Option<Vec<crate::engine_config::EngineConfig>>,
501    initialization: Arc<Mutex<()>>,
502}
503
504impl EmbedderRegistry {
505    /// Create an empty registry.
506    pub fn new() -> Self {
507        Self {
508            entries: HashMap::new(),
509            ordered_names: Vec::new(),
510            bound_engines: None,
511            initialization: Arc::new(Mutex::new(())),
512        }
513    }
514
515    /// Register a provider, preserving its position for later name enumeration.
516    ///
517    /// Before a provider is configured-bound or selected for resolution,
518    /// registration with the same name replaces it in place. Afterwards,
519    /// duplicate registration leaves the bound or serving provider unchanged.
520    pub fn register<P: EmbedderProvider + 'static>(&mut self, provider: P) -> RuntimeResult<()> {
521        self.insert(provider, false)
522    }
523
524    /// Register the runtime-owned lattice adapter whose passage preparation is
525    /// audited. This is deliberately not available to pack providers.
526    pub(crate) fn register_builtin(&mut self, provider: LatticeEmbedderProvider) {
527        self.insert(provider, true)
528            .expect("configured built-in embedding models must have unique names");
529    }
530
531    /// Test-only attested provider. The wrapper below owns passage preparation,
532    /// so a fake backend cannot change the bytes whose digest is recorded.
533    #[cfg(feature = "test-internals")]
534    pub fn register_test_audited<P: EmbedderProvider + 'static>(
535        &mut self,
536        model: EmbeddingModel,
537        provider: P,
538    ) {
539        assert_eq!(provider.name(), model.to_string());
540        self.insert(TestAuditedProvider { provider }, true)
541            .expect("test audited provider names must be unique");
542    }
543
544    /// Test-only replacement of a provider whose name is already bound.
545    ///
546    /// A test that builds its host through the production startup path finds
547    /// the configured names bound before it can register a deterministic
548    /// backend, and [`Self::register`] refuses a bound name. This swaps the
549    /// provider under such a name and leaves the bound engine list unchanged.
550    /// It refuses a name that is not bound, a provider that is already
551    /// serving, and dimensions that differ from the bound provider's.
552    #[cfg(feature = "test-internals")]
553    pub fn replace_bound_for_test<P: EmbedderProvider + 'static>(
554        &mut self,
555        provider: P,
556    ) -> RuntimeResult<()> {
557        let name = provider.name().to_owned();
558        if !self
559            .bound_engines
560            .as_ref()
561            .is_some_and(|engines| engines.iter().any(|engine| engine.name == name))
562        {
563            return Err(RuntimeError::InvalidInput(format!(
564                "embedding provider `{name}` is not bound to configured engines"
565            )));
566        }
567        let bound = self.entries.get(&name).ok_or_else(|| {
568            RuntimeError::InvalidInput(format!("embedding provider `{name}` is not registered"))
569        })?;
570        if bound.serving.load(Ordering::Acquire) {
571            return Err(RuntimeError::InvalidInput(format!(
572                "embedding provider `{name}` is already serving"
573            )));
574        }
575        let expected = bound.provider.dimensions();
576        if provider.dimensions() != expected {
577            return Err(RuntimeError::InvalidInput(format!(
578                "embedding provider `{name}` has dimensions {}, but the bound provider has {expected}",
579                provider.dimensions()
580            )));
581        }
582        self.entries.insert(
583            name,
584            EmbedderEntry {
585                provider: Arc::new(provider),
586                cell: Arc::new(OnceCell::new()),
587                serving: Arc::new(AtomicBool::new(false)),
588                audited_document_preparation: false,
589            },
590        );
591        Ok(())
592    }
593
594    fn insert<P: EmbedderProvider + 'static>(
595        &mut self,
596        provider: P,
597        audited_document_preparation: bool,
598    ) -> RuntimeResult<()> {
599        let name = provider.name().to_owned();
600        if self
601            .bound_engines
602            .as_ref()
603            .is_some_and(|engines| engines.iter().any(|engine| engine.name == name))
604        {
605            return Err(RuntimeError::InvalidInput(format!(
606                "embedding provider `{name}` is already bound to configured engines"
607            )));
608        }
609        if self
610            .entries
611            .get(&name)
612            .is_some_and(|entry| entry.serving.load(Ordering::Acquire))
613        {
614            return Err(RuntimeError::InvalidInput(format!(
615                "embedding provider `{name}` is already serving"
616            )));
617        }
618        let entry = EmbedderEntry {
619            provider: Arc::new(provider),
620            cell: Arc::new(OnceCell::new()),
621            serving: Arc::new(AtomicBool::new(false)),
622            audited_document_preparation,
623        };
624        if self.entries.contains_key(&name) {
625            self.entries.insert(name, entry);
626        } else {
627            self.ordered_names.push(name.clone());
628            self.entries.insert(name, entry);
629        }
630        Ok(())
631    }
632
633    /// Look up a provider by name.
634    pub fn get_provider(&self, name: &str) -> Option<&dyn EmbedderProvider> {
635        self.entries.get(name).map(|e| e.provider.as_ref())
636    }
637
638    /// Returns `true` if a provider with this name is registered.
639    pub fn contains(&self, name: &str) -> bool {
640        self.entries.contains_key(name)
641    }
642
643    /// Names of all registered providers, in registration order.
644    pub fn names(&self) -> Vec<String> {
645        self.ordered_names.clone()
646    }
647
648    pub(crate) fn initialization_lock(&self) -> Arc<Mutex<()>> {
649        Arc::clone(&self.initialization)
650    }
651
652    pub(crate) fn bound_engines(&self) -> Option<&[crate::engine_config::EngineConfig]> {
653        self.bound_engines.as_deref()
654    }
655
656    pub(crate) fn bind_engines(
657        &mut self,
658        engines: &[crate::engine_config::EngineConfig],
659    ) -> RuntimeResult<()> {
660        if let Some(bound) = &self.bound_engines {
661            return if bound == engines {
662                Ok(())
663            } else {
664                Err(RuntimeError::InvalidInput(
665                    "configured embedding engines are already bound to a different ordered list"
666                        .into(),
667                ))
668            };
669        }
670        for engine in engines {
671            let provider = self.get_provider(&engine.name).ok_or_else(|| {
672                RuntimeError::InvalidInput(format!(
673                    "embedding provider `{}` is not registered at startup",
674                    engine.name
675                ))
676            })?;
677            let actual = provider.dimensions();
678            if actual == 0 || u32::try_from(actual).is_err() {
679                return Err(RuntimeError::InvalidInput(format!(
680                    "embedding provider `{}` has invalid dimensions {actual}; expected 1..=4294967295",
681                    engine.name
682                )));
683            }
684            if let Some(model) = crate::config::parse_embedding_model_alias(&engine.name) {
685                if actual != model.dimensions() {
686                    return Err(RuntimeError::InvalidInput(format!(
687                        "embedding provider `{}` has dimensions {actual}, but its built-in storage binding requires {}",
688                        engine.name,
689                        model.dimensions()
690                    )));
691                }
692            }
693            engine
694                .check_dimensions(actual)
695                .map_err(|error| RuntimeError::InvalidInput(error.to_string()))?;
696        }
697        self.bound_engines = Some(engines.to_vec());
698        Ok(())
699    }
700
701    /// Return a cloned entry for `name` without holding any lock.
702    ///
703    /// The caller can then call [`EmbedderEntry::resolve`] without holding
704    /// a lock — this avoids holding a `RwLockGuard` across `await` points.
705    /// It marks the provider as selected for serving so registration cannot
706    /// replace it while a request is resolving the service. Returns `None` if
707    /// `name` is not registered.
708    pub(crate) fn get_entry(&self, name: &str) -> Option<EmbedderEntry> {
709        let entry = self.entries.get(name)?;
710        entry.serving.store(true, Ordering::Release);
711        Some(entry.clone())
712    }
713
714    /// Lazily resolve a registered provider to its live [`EmbeddingService`].
715    ///
716    /// Returns [`RuntimeError::UnknownModel`] if `name` is not registered.
717    /// The first call for a given name triggers [`EmbedderProvider::build`];
718    /// subsequent calls return the cached `Arc`.
719    ///
720    /// Prefer [`crate::KhiveRuntime::embedder`] over calling this directly from pack
721    /// handlers — the runtime method handles alias resolution and error mapping.
722    pub async fn get_service(&self, name: &str) -> RuntimeResult<Arc<dyn EmbeddingService>> {
723        let entry = self
724            .entries
725            .get(name)
726            .ok_or_else(|| RuntimeError::UnknownModel(name.to_string()))?
727            .clone();
728        entry.serving.store(true, Ordering::Release);
729
730        Ok(entry.resolve().await?.0)
731    }
732}
733
734#[cfg(feature = "test-internals")]
735struct TestAuditedProvider<P> {
736    provider: P,
737}
738
739#[cfg(feature = "test-internals")]
740#[async_trait]
741impl<P: EmbedderProvider> EmbedderProvider for TestAuditedProvider<P> {
742    fn name(&self) -> &str {
743        self.provider.name()
744    }
745
746    fn dimensions(&self) -> usize {
747        self.provider.dimensions()
748    }
749
750    async fn build(&self) -> RuntimeResult<Arc<dyn EmbeddingService>> {
751        Ok(Arc::new(TestAuditedService(self.provider.build().await?)))
752    }
753}
754
755/// Deliberately does not delegate `embed_passage`: the lattice trait default
756/// applies the model's document instruction before calling `embed`, exactly
757/// as the audited built-in path does. Test providers supply only fake vectors.
758#[cfg(feature = "test-internals")]
759struct TestAuditedService(Arc<dyn EmbeddingService>);
760
761#[cfg(feature = "test-internals")]
762#[async_trait]
763impl EmbeddingService for TestAuditedService {
764    async fn embed(
765        &self,
766        texts: &[String],
767        model: EmbeddingModel,
768    ) -> lattice_embed::Result<Vec<Vec<f32>>> {
769        self.0.embed(texts, model).await
770    }
771
772    fn supports_model(&self, model: EmbeddingModel) -> bool {
773        self.0.supports_model(model)
774    }
775
776    fn name(&self) -> &'static str {
777        self.0.name()
778    }
779}
780
781impl EmbedderEntry {
782    pub(crate) fn cached_service(&self) -> Option<Arc<dyn EmbeddingService>> {
783        self.cell.get().map(Arc::clone)
784    }
785
786    pub(crate) fn has_audited_document_preparation(&self) -> bool {
787        self.audited_document_preparation
788    }
789
790    /// Lazily initialise and return the embedding service for this entry.
791    ///
792    /// `OnceCell::get_or_try_init` single-flights concurrent cold callers: only
793    /// one of them runs [`EmbedderProvider::build`], and every other waiter
794    /// receives the same result once it completes. Only the caller whose task
795    /// actually ran `build()` gets back `Some(duration)`; every other caller
796    /// (cache hit or wait-for-in-flight-build) gets `None`.
797    ///
798    /// Returns `RuntimeError` if `build()` fails, rather than panicking. On
799    /// failure the cell stays unset, so a later call retries the build.
800    pub(crate) async fn resolve(self) -> RuntimeResult<(Arc<dyn EmbeddingService>, Option<i64>)> {
801        let mut own_init_duration_us: Option<i64> = None;
802        let provider = Arc::clone(&self.provider);
803        let init_duration_us = &mut own_init_duration_us;
804        let svc = self
805            .cell
806            .get_or_try_init(|| async move {
807                let init_start = std::time::Instant::now();
808                let svc = provider.build().await.map_err(|e| {
809                    crate::error::RuntimeError::Internal(format!(
810                        "EmbedderProvider '{}' build() failed: {e}",
811                        provider.name()
812                    ))
813                })?;
814                *init_duration_us = Some(init_start.elapsed().as_micros() as i64);
815                Ok::<_, RuntimeError>(svc)
816            })
817            .await?;
818        Ok((Arc::clone(svc), own_init_duration_us))
819    }
820}
821
822// ── LatticeEmbedderProvider ───────────────────────────────────────────────────
823
824/// Adapter that wraps a [`lattice_embed::EmbeddingModel`] as an
825/// [`EmbedderProvider`].
826///
827/// All built-in models (MiniLM, paraphrase-multilingual, BGE variants, etc.)
828/// are registered as `LatticeEmbedderProvider` instances during
829/// `KhiveRuntime` construction. External callers do not need to use this type
830/// unless they are constructing a custom registry from scratch.
831pub struct LatticeEmbedderProvider {
832    model: EmbeddingModel,
833    /// Cached `to_string()` result so `name()` can return `&str`.
834    name: String,
835}
836
837impl LatticeEmbedderProvider {
838    /// Create a new provider wrapping the given lattice model.
839    pub fn new(model: EmbeddingModel) -> Self {
840        let name = model.to_string();
841        Self { model, name }
842    }
843}
844
845#[async_trait]
846impl EmbedderProvider for LatticeEmbedderProvider {
847    fn name(&self) -> &str {
848        &self.name
849    }
850
851    fn dimensions(&self) -> usize {
852        self.model.dimensions()
853    }
854
855    async fn build(&self) -> RuntimeResult<Arc<dyn EmbeddingService>> {
856        let native = Arc::new(NativeEmbeddingService::with_model(self.model));
857        native.ensure_loaded().await?;
858        // Embedder loading frees transient heap that glibc would otherwise retain.
859        #[cfg(all(target_os = "linux", target_env = "gnu"))]
860        // SAFETY: malloc_trim has no preconditions. Its 0/1 release status is intentionally ignored.
861        unsafe {
862            libc::malloc_trim(0);
863        }
864        Ok(cached_blocking_service(native))
865    }
866}
867
868/// Keep result-cache lookup outside worker admission for every retrieval role.
869fn cached_blocking_service<S: EmbeddingService + 'static>(
870    inner: Arc<S>,
871) -> Arc<dyn EmbeddingService> {
872    let blocking = Arc::new(BlockingEmbeddingService::new(inner));
873    Arc::new(CachedEmbeddingService::with_default_cache(blocking))
874}
875
876#[cfg(test)]
877#[path = "embedding_cache_admission_tests.rs"]
878mod cache_admission_tests;
879
880#[cfg(test)]
881#[path = "embedding_token_count_tests.rs"]
882mod token_count_tests;
883
884// ── Unit tests ────────────────────────────────────────────────────────────────
885
886#[cfg(test)]
887mod tests {
888    use super::*;
889    use std::collections::HashSet;
890    use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
891    use std::sync::{Condvar, Mutex};
892    use std::time::Duration;
893    use tokio::sync::Notify;
894
895    struct ConstVecProvider {
896        name: String,
897        dims: usize,
898        build_calls: Arc<AtomicUsize>,
899    }
900
901    impl ConstVecProvider {
902        fn new(name: &str, dims: usize) -> Self {
903            Self {
904                name: name.to_owned(),
905                dims,
906                build_calls: Arc::new(AtomicUsize::new(0)),
907            }
908        }
909    }
910
911    /// A trivial embedding service that returns a constant vector of `1.0`s.
912    /// The `model` parameter is ignored — this service always returns the
913    /// same synthetic vector regardless of which model is requested.
914    pub(super) struct ConstVecService {
915        pub(super) dims: usize,
916    }
917
918    #[async_trait]
919    impl EmbeddingService for ConstVecService {
920        async fn embed(
921            &self,
922            texts: &[String],
923            _model: EmbeddingModel,
924        ) -> std::result::Result<Vec<Vec<f32>>, lattice_embed::EmbedError> {
925            Ok(texts.iter().map(|_| vec![1.0_f32; self.dims]).collect())
926        }
927
928        fn supports_model(&self, _model: EmbeddingModel) -> bool {
929            true
930        }
931
932        fn name(&self) -> &'static str {
933            "const-vec-service"
934        }
935    }
936
937    #[async_trait]
938    impl EmbedderProvider for ConstVecProvider {
939        fn name(&self) -> &str {
940            &self.name
941        }
942
943        fn dimensions(&self) -> usize {
944            self.dims
945        }
946
947        async fn build(&self) -> RuntimeResult<Arc<dyn EmbeddingService>> {
948            self.build_calls.fetch_add(1, Ordering::SeqCst);
949            Ok(Arc::new(ConstVecService { dims: self.dims }))
950        }
951    }
952
953    #[test]
954    fn builtin_provider_can_be_replaced_before_resolution_without_reordering() {
955        let model = EmbeddingModel::MultilingualE5Small;
956        let name = model.to_string();
957        let mut registry = EmbedderRegistry::new();
958        registry.register_builtin(LatticeEmbedderProvider::new(model));
959        let original_names = registry.names();
960
961        registry
962            .register(ConstVecProvider::new(&name, model.dimensions()))
963            .expect("pre-serving replacement should succeed");
964        assert_eq!(registry.names(), original_names);
965        assert!(!registry
966            .get_entry(&name)
967            .expect("replacement entry")
968            .has_audited_document_preparation());
969    }
970
971    struct FirstLoadBlockingService {
972        loaded: AtomicBool,
973    }
974
975    #[async_trait]
976    impl EmbeddingService for FirstLoadBlockingService {
977        async fn embed(
978            &self,
979            texts: &[String],
980            _model: EmbeddingModel,
981        ) -> lattice_embed::Result<Vec<Vec<f32>>> {
982            if !self.loaded.swap(true, Ordering::SeqCst) {
983                tokio::task::spawn_blocking(|| {})
984                    .await
985                    .map_err(|error| lattice_embed::EmbedError::Internal(error.to_string()))?;
986            }
987            Ok(texts.iter().map(|_| vec![1.0]).collect())
988        }
989
990        fn supports_model(&self, _model: EmbeddingModel) -> bool {
991            true
992        }
993
994        fn name(&self) -> &'static str {
995            "first-load-blocking-service"
996        }
997    }
998
999    pub(super) struct BlockingTestService {
1000        pub(super) calls: Mutex<Vec<String>>,
1001        pub(super) entered: AtomicUsize,
1002        release: (Mutex<bool>, Condvar),
1003        pub(super) thread_ids: Mutex<HashSet<std::thread::ThreadId>>,
1004    }
1005
1006    impl BlockingTestService {
1007        pub(super) fn new() -> Self {
1008            Self {
1009                calls: Mutex::new(Vec::new()),
1010                entered: AtomicUsize::new(0),
1011                release: (Mutex::new(false), Condvar::new()),
1012                thread_ids: Mutex::new(HashSet::new()),
1013            }
1014        }
1015
1016        pub(super) fn release(&self) {
1017            *self
1018                .release
1019                .0
1020                .lock()
1021                .expect("release lock must not be poisoned") = true;
1022            self.release.1.notify_all();
1023        }
1024    }
1025
1026    #[async_trait]
1027    impl EmbeddingService for BlockingTestService {
1028        async fn embed(
1029            &self,
1030            texts: &[String],
1031            _model: EmbeddingModel,
1032        ) -> lattice_embed::Result<Vec<Vec<f32>>> {
1033            let text = texts.first().cloned().unwrap_or_default();
1034            self.thread_ids
1035                .lock()
1036                .expect("thread id lock must not be poisoned")
1037                .insert(std::thread::current().id());
1038            self.calls
1039                .lock()
1040                .expect("call lock must not be poisoned")
1041                .push(text.clone());
1042            self.entered.fetch_add(1, Ordering::Release);
1043
1044            if text != "later" {
1045                let (released, wake) = &self.release;
1046                let guard = released.lock().expect("release lock must not be poisoned");
1047                let _guard = wake
1048                    .wait_while(guard, |released| !*released)
1049                    .expect("release lock must not be poisoned");
1050            }
1051
1052            Ok(texts.iter().map(|_| vec![1.0]).collect())
1053        }
1054
1055        async fn embed_query(
1056            &self,
1057            texts: &[String],
1058            model: EmbeddingModel,
1059        ) -> lattice_embed::Result<Vec<Vec<f32>>> {
1060            // This custom service owns its inputs; a built-in instruction must
1061            // not rewrite the sentinel that releases its synthetic worker.
1062            self.embed(texts, model).await
1063        }
1064
1065        fn supports_model(&self, _model: EmbeddingModel) -> bool {
1066            true
1067        }
1068
1069        fn name(&self) -> &'static str {
1070            "blocking-test-service"
1071        }
1072    }
1073
1074    #[test]
1075    fn blocking_adapter_first_use_completes_with_single_blocking_thread() {
1076        let runtime = tokio::runtime::Builder::new_current_thread()
1077            .enable_time()
1078            .max_blocking_threads(1)
1079            .build()
1080            .expect("current-thread runtime must build");
1081        let service = BlockingEmbeddingService::new(Arc::new(FirstLoadBlockingService {
1082            loaded: AtomicBool::new(false),
1083        }));
1084
1085        let result = runtime.block_on(async {
1086            tokio::time::timeout(
1087                Duration::from_secs(5),
1088                service.embed(&["first use".to_owned()], EmbeddingModel::default()),
1089            )
1090            .await
1091        });
1092        runtime.shutdown_timeout(Duration::from_secs(1));
1093
1094        let embeddings = result
1095            .expect("first-use embedding must not exhaust the blocking pool")
1096            .expect("first-use embedding must succeed");
1097        assert_eq!(embeddings, vec![vec![1.0]]);
1098    }
1099
1100    #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
1101    async fn blocking_adapter_uses_one_worker_for_concurrent_calls() {
1102        const CALL_COUNT: usize = 32;
1103        let inner = Arc::new(BlockingTestService::new());
1104        let service = Arc::new(BlockingEmbeddingService::new(Arc::clone(&inner)));
1105        let mut calls = Vec::with_capacity(CALL_COUNT);
1106
1107        for index in 0..CALL_COUNT {
1108            let service = Arc::clone(&service);
1109            calls.push(tokio::spawn(async move {
1110                service
1111                    .embed(&[format!("request-{index}")], EmbeddingModel::default())
1112                    .await
1113            }));
1114        }
1115
1116        let _ = tokio::time::timeout(Duration::from_millis(250), async {
1117            while inner.entered.load(Ordering::Acquire) < 2 {
1118                tokio::task::yield_now().await;
1119            }
1120        })
1121        .await;
1122        inner.release();
1123
1124        for call in calls {
1125            call.await
1126                .expect("embedding task must not panic")
1127                .expect("embedding call must succeed");
1128        }
1129        assert_eq!(
1130            inner
1131                .thread_ids
1132                .lock()
1133                .expect("thread id lock must not be poisoned")
1134                .len(),
1135            1,
1136            "concurrent calls must share one native worker thread"
1137        );
1138    }
1139
1140    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1141    async fn blocking_adapter_skips_timed_out_call_and_serves_later_call() {
1142        let inner = Arc::new(BlockingTestService::new());
1143        let service = Arc::new(BlockingEmbeddingService::new(Arc::clone(&inner)));
1144
1145        let first_service = Arc::clone(&service);
1146        let first = tokio::spawn(async move {
1147            first_service
1148                .embed(&["first".to_owned()], EmbeddingModel::default())
1149                .await
1150        });
1151        tokio::time::timeout(Duration::from_secs(1), async {
1152            while inner.entered.load(Ordering::Acquire) == 0 {
1153                tokio::task::yield_now().await;
1154            }
1155        })
1156        .await
1157        .expect("first embedding call must enter the native service");
1158
1159        let abandoned = tokio::time::timeout(
1160            Duration::from_millis(50),
1161            service.embed(&["abandoned".to_owned()], EmbeddingModel::default()),
1162        )
1163        .await;
1164        let later_service = Arc::clone(&service);
1165        let later = tokio::spawn(async move {
1166            later_service
1167                .embed(&["later".to_owned()], EmbeddingModel::default())
1168                .await
1169        });
1170        inner.release();
1171
1172        first
1173            .await
1174            .expect("first embedding task must not panic")
1175            .expect("first embedding call must succeed");
1176        let later_result = tokio::time::timeout(Duration::from_secs(1), later)
1177            .await
1178            .expect("later embedding call must be served")
1179            .expect("later embedding task must not panic")
1180            .expect("later embedding call must succeed");
1181
1182        assert!(abandoned.is_err(), "queued embedding call must time out");
1183        assert_eq!(later_result, vec![vec![1.0]]);
1184        assert_eq!(
1185            *inner.calls.lock().expect("call lock must not be poisoned"),
1186            vec!["first".to_owned(), "later".to_owned()],
1187            "the worker must skip a queued call whose receiver is closed"
1188        );
1189    }
1190
1191    pub(super) struct ReleaseWorkerOnDrop(pub(super) Arc<BlockingTestService>);
1192
1193    impl Drop for ReleaseWorkerOnDrop {
1194        fn drop(&mut self) {
1195            self.0.release();
1196        }
1197    }
1198
1199    struct ServiceProvider(Arc<dyn EmbeddingService>);
1200
1201    #[async_trait]
1202    impl EmbedderProvider for ServiceProvider {
1203        fn name(&self) -> &str {
1204            "queue-test"
1205        }
1206        fn dimensions(&self) -> usize {
1207            1
1208        }
1209        async fn build(&self) -> RuntimeResult<Arc<dyn EmbeddingService>> {
1210            Ok(Arc::clone(&self.0))
1211        }
1212    }
1213
1214    pub(super) fn queue_runtime(service: Arc<dyn EmbeddingService>) -> crate::KhiveRuntime {
1215        let runtime = crate::KhiveRuntime::memory().expect("memory runtime");
1216        runtime.register_embedder(ServiceProvider(service));
1217        runtime
1218    }
1219
1220    pub(super) async fn poll_once<F: std::future::Future + ?Sized>(
1221        mut future: std::pin::Pin<&mut F>,
1222    ) -> std::task::Poll<F::Output> {
1223        std::future::poll_fn(|cx| std::task::Poll::Ready(future.as_mut().poll(cx))).await
1224    }
1225
1226    pub(super) async fn wait_for_entered(inner: &BlockingTestService, count: usize) {
1227        let watchdog = std::time::Instant::now() + Duration::from_secs(2);
1228        while inner.entered.load(Ordering::Acquire) < count {
1229            assert!(
1230                std::time::Instant::now() < watchdog,
1231                "setup watchdog: native worker did not enter"
1232            );
1233            tokio::task::yield_now().await;
1234        }
1235    }
1236
1237    async fn drive_until_entered<F: std::future::Future + ?Sized>(
1238        mut future: std::pin::Pin<&mut F>,
1239        inner: &BlockingTestService,
1240        count: usize,
1241    ) {
1242        let watchdog = std::time::Instant::now() + Duration::from_secs(2);
1243        while inner.entered.load(Ordering::Acquire) < count {
1244            assert!(
1245                std::time::Instant::now() < watchdog,
1246                "setup watchdog: driven runtime call did not reach the native worker"
1247            );
1248            assert!(
1249                poll_once(future.as_mut()).await.is_pending(),
1250                "held inference must remain pending while the runtime call is driven"
1251            );
1252            tokio::task::yield_now().await;
1253        }
1254    }
1255
1256    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1257    async fn runtime_embedding_waits_for_capacity_and_drains_n_plus_one_calls() {
1258        let inner = Arc::new(BlockingTestService::new());
1259        let _release = ReleaseWorkerOnDrop(Arc::clone(&inner));
1260        let service = Arc::new(BlockingEmbeddingService::new(Arc::clone(&inner)));
1261        let runtime = queue_runtime(service.clone());
1262        runtime
1263            .embedder("queue-test")
1264            .await
1265            .expect("resolve provider before the admission fixture");
1266        let first_text = vec!["first".to_owned()];
1267        let mut first = Box::pin(runtime.embed_batch_with_model("queue-test", &first_text));
1268        assert!(poll_once(first.as_mut()).await.is_pending());
1269        drive_until_entered(first.as_mut(), &inner, 1).await;
1270
1271        let texts: Vec<_> = (0..EMBEDDING_QUEUE_CAPACITY)
1272            .map(|index| vec![format!("queued-{index}")])
1273            .collect();
1274        let mut queued = Vec::new();
1275        for text in &texts {
1276            let mut call = Box::pin(runtime.embed_batch_with_model("queue-test", text));
1277            assert!(
1278                poll_once(call.as_mut()).await.is_pending(),
1279                "the bounded queue must admit its N actual runtime calls"
1280            );
1281            queued.push(call);
1282        }
1283        assert_eq!(
1284            service.in_flight_bytes.load(Ordering::Acquire),
1285            first_text.iter().map(String::len).sum::<usize>()
1286                + texts.iter().flatten().map(String::len).sum::<usize>(),
1287            "the held worker and N queued runtime jobs must own their exact input bytes"
1288        );
1289        let overflow_text = vec!["overflow".to_owned()];
1290        let mut overflow = Box::pin(runtime.embed_batch_with_model("queue-test", &overflow_text));
1291        let overflow_before_release = poll_once(overflow.as_mut()).await;
1292        inner.release();
1293        assert!(
1294            overflow_before_release.is_pending(),
1295            "request N+1 must wait for a slot instead of failing: {overflow_before_release:?}"
1296        );
1297        assert_eq!(first.await.unwrap(), vec![vec![1.0]]);
1298        for call in queued {
1299            assert_eq!(call.await.unwrap(), vec![vec![1.0]]);
1300        }
1301        assert_eq!(overflow.await.unwrap(), vec![vec![1.0]]);
1302        assert_eq!(
1303            inner.entered.load(Ordering::Acquire),
1304            EMBEDDING_QUEUE_CAPACITY + 2
1305        );
1306        assert_eq!(inner.thread_ids.lock().unwrap().len(), 1);
1307    }
1308
1309    #[tokio::test(start_paused = true)]
1310    async fn runtime_embedding_expired_admission_is_retryable_and_never_enqueued() {
1311        let inner = Arc::new(BlockingTestService::new());
1312        let _release = ReleaseWorkerOnDrop(Arc::clone(&inner));
1313        let service = Arc::new(BlockingEmbeddingService::new(Arc::clone(&inner)));
1314        let runtime = queue_runtime(service.clone());
1315        runtime
1316            .embedder("queue-test")
1317            .await
1318            .expect("resolve provider before the admission fixture");
1319        let first_text = vec!["first".to_owned()];
1320        let mut first = Box::pin(runtime.embed_batch_with_model("queue-test", &first_text));
1321        assert!(poll_once(first.as_mut()).await.is_pending());
1322        drive_until_entered(first.as_mut(), &inner, 1).await;
1323        let texts: Vec<_> = (0..EMBEDDING_QUEUE_CAPACITY)
1324            .map(|index| vec![format!("queued-{index}")])
1325            .collect();
1326        let mut queued = Vec::new();
1327        for text in &texts {
1328            let mut call = Box::pin(runtime.embed_batch_with_model("queue-test", text));
1329            assert!(poll_once(call.as_mut()).await.is_pending());
1330            queued.push(call);
1331        }
1332        assert_eq!(
1333            service.in_flight_bytes.load(Ordering::Acquire),
1334            first_text.iter().map(String::len).sum::<usize>()
1335                + texts.iter().flatten().map(String::len).sum::<usize>(),
1336            "the held worker and N queued runtime jobs must own their exact input bytes"
1337        );
1338        let deadline = khive_storage::RequestReadDeadline::after(Duration::from_millis(100));
1339        let mut expired = Box::pin(khive_storage::scope_request_read_deadline_at(
1340            deadline,
1341            runtime.embed_with_model("queue-test", "expired"),
1342        ));
1343        assert!(
1344            poll_once(expired.as_mut()).await.is_pending(),
1345            "an unexpired full-queue call must wait"
1346        );
1347        tokio::time::advance(Duration::from_millis(100)).await;
1348        let result = poll_once(expired.as_mut()).await;
1349        inner.release();
1350        let error = match result {
1351            std::task::Poll::Ready(Err(error)) => error,
1352            other => panic!("expired admission must produce a typed runtime error: {other:?}"),
1353        };
1354        assert!(
1355            matches!(&error, RuntimeError::Storage(khive_storage::StorageError::AdmissionTimeout {
1356            operation, timeout_ms: 100, pool_identity: None,
1357        }) if operation == "embedding admission"),
1358            "{error:?}"
1359        );
1360        assert!(
1361            error.retryable_failure_context().is_some(),
1362            "pre-admission expiry must use the existing retryable classification"
1363        );
1364        let projected = crate::error_projection::runtime_error_value(
1365            error,
1366            crate::DomainDisposition::NotCommitted,
1367        );
1368        assert_eq!(projected["retryable"], true);
1369        assert_eq!(projected["operation"], "embedding admission");
1370        assert_eq!(first.await.unwrap(), vec![vec![1.0]]);
1371        for call in queued {
1372            assert_eq!(call.await.unwrap(), vec![vec![1.0]]);
1373        }
1374        assert!(
1375            !inner
1376                .calls
1377                .lock()
1378                .unwrap()
1379                .iter()
1380                .any(|text| text == "expired"),
1381            "an expired waiter must never reach inference"
1382        );
1383    }
1384
1385    #[tokio::test(start_paused = true)]
1386    async fn runtime_embedding_earlier_absolute_bound_is_not_renewed_when_slot_is_ready() {
1387        let inner = Arc::new(BlockingTestService::new());
1388        let _release = ReleaseWorkerOnDrop(Arc::clone(&inner));
1389        let runtime = queue_runtime(Arc::new(BlockingEmbeddingService::new(Arc::clone(&inner))));
1390        runtime
1391            .embedder("queue-test")
1392            .await
1393            .expect("resolve provider before the admission deadline");
1394        let deadline = khive_storage::RequestReadDeadline::after(Duration::from_millis(100));
1395        tokio::time::advance(Duration::from_millis(100)).await;
1396        let mut call = Box::pin(khive_storage::scope_request_read_deadline_at(
1397            deadline,
1398            runtime.embed_with_model("queue-test", "expired-ready-slot"),
1399        ));
1400        let result = poll_once(call.as_mut()).await;
1401        inner.release();
1402        assert!(
1403            matches!(
1404                result,
1405                std::task::Poll::Ready(Err(RuntimeError::Storage(
1406                    khive_storage::StorageError::AdmissionTimeout { timeout_ms: 0, .. }
1407                )))
1408            ),
1409            "an already-expired original bound must refuse even an available slot: {result:?}"
1410        );
1411        assert_eq!(
1412            inner.entered.load(Ordering::Acquire),
1413            0,
1414            "capacity becoming ready must not bypass expiry"
1415        );
1416    }
1417
1418    #[tokio::test(start_paused = true)]
1419    async fn runtime_embedding_all_six_call_families_refuse_expired_admission() {
1420        let inner = Arc::new(BlockingTestService::new());
1421        let _release = ReleaseWorkerOnDrop(Arc::clone(&inner));
1422        let runtime = queue_runtime(Arc::new(BlockingEmbeddingService::new(Arc::clone(&inner))));
1423        let texts = vec!["later".to_owned()];
1424        for family in 0..6 {
1425            let result = khive_storage::scope_request_read_deadline(Duration::ZERO, async {
1426                match family {
1427                    0 => runtime.embed_with_model("queue-test", "later").await,
1428                    1 => runtime
1429                        .embed_document_with_model_outcome("queue-test", "later")
1430                        .await
1431                        .map(|outcome| outcome.vector),
1432                    2 => runtime.embed_query_with_model("queue-test", "later").await,
1433                    3 => runtime
1434                        .embed_batch_with_model("queue-test", &texts)
1435                        .await
1436                        .map(|vectors| vectors[0].clone()),
1437                    4 => runtime
1438                        .embed_document_batch_with_model("queue-test", &texts)
1439                        .await
1440                        .map(|vectors| vectors[0].clone()),
1441                    _ => runtime
1442                        .embed_query_batch_with_model("queue-test", &texts)
1443                        .await
1444                        .map(|vectors| vectors[0].clone()),
1445                }
1446            })
1447            .await;
1448            assert!(
1449                matches!(
1450                    result,
1451                    Err(RuntimeError::Storage(
1452                        khive_storage::StorageError::AdmissionTimeout { .. }
1453                    ))
1454                ),
1455                "family {family} must reach the actual runtime admission boundary: {result:?}"
1456            );
1457        }
1458        assert_eq!(inner.entered.load(Ordering::Acquire), 0);
1459    }
1460
1461    #[tokio::test(start_paused = true)]
1462    async fn runtime_embedding_admitted_inference_is_not_reported_as_admission_timeout() {
1463        let inner = Arc::new(BlockingTestService::new());
1464        let _release = ReleaseWorkerOnDrop(Arc::clone(&inner));
1465        let runtime = queue_runtime(Arc::new(BlockingEmbeddingService::new(Arc::clone(&inner))));
1466        runtime
1467            .embedder("queue-test")
1468            .await
1469            .expect("resolve provider before the admission fixture");
1470        let mut call = Box::pin(khive_storage::scope_request_read_deadline(
1471            Duration::from_millis(100),
1472            runtime.embed_with_model("queue-test", "first"),
1473        ));
1474        assert!(poll_once(call.as_mut()).await.is_pending());
1475        drive_until_entered(call.as_mut(), &inner, 1).await;
1476        tokio::time::advance(Duration::from_millis(100)).await;
1477        let after_deadline = poll_once(call.as_mut()).await;
1478        inner.release();
1479        assert!(after_deadline.is_pending(),
1480            "already-running inference cannot be called a pre-admission refusal: {after_deadline:?}");
1481        assert_eq!(call.await.unwrap(), vec![1.0]);
1482    }
1483
1484    #[tokio::test(start_paused = true)]
1485    async fn runtime_embedding_second_job_keeps_the_original_admission_bound() {
1486        let inner = Arc::new(BlockingTestService::new());
1487        let _release = ReleaseWorkerOnDrop(Arc::clone(&inner));
1488        let service = BlockingEmbeddingService::new(Arc::clone(&inner));
1489        let first = vec!["first".to_owned()];
1490        let second = vec!["second".to_owned()];
1491        let mut call = Box::pin(khive_storage::scope_request_read_deadline(
1492            Duration::from_millis(100),
1493            with_embedding_admission(async {
1494                service.embed(&first, EmbeddingModel::default()).await?;
1495                service.embed(&second, EmbeddingModel::default()).await
1496            }),
1497        ));
1498        assert!(poll_once(call.as_mut()).await.is_pending());
1499        drive_until_entered(call.as_mut(), &inner, 1).await;
1500        tokio::time::advance(Duration::from_millis(100)).await;
1501        assert!(
1502            poll_once(call.as_mut()).await.is_pending(),
1503            "the admitted first job must finish despite expiry"
1504        );
1505        inner.release();
1506
1507        // A wall-clock watchdog keeps the paused clock from racing worker completion.
1508        let watchdog = std::time::Instant::now() + Duration::from_secs(2);
1509        let result = loop {
1510            if let std::task::Poll::Ready(result) = poll_once(call.as_mut()).await {
1511                break result;
1512            }
1513            assert!(
1514                std::time::Instant::now() < watchdog,
1515                "second-job admission must settle instead of waiting forever"
1516            );
1517            tokio::task::yield_now().await;
1518        };
1519        assert!(
1520            matches!(
1521                result,
1522                Err(RuntimeError::Storage(
1523                    khive_storage::StorageError::AdmissionTimeout {
1524                        timeout_ms: 100,
1525                        ..
1526                    }
1527                ))
1528            ),
1529            "the second job must use the original bound: {result:?}"
1530        );
1531        assert_eq!(*inner.calls.lock().unwrap(), vec!["first".to_owned()]);
1532        assert_eq!(inner.entered.load(Ordering::Acquire), 1);
1533    }
1534
1535    struct CustomWaitingService {
1536        release: Notify,
1537    }
1538
1539    #[async_trait]
1540    impl EmbeddingService for CustomWaitingService {
1541        async fn embed(
1542            &self,
1543            texts: &[String],
1544            _model: EmbeddingModel,
1545        ) -> lattice_embed::Result<Vec<Vec<f32>>> {
1546            self.release.notified().await;
1547            Ok(texts.iter().map(|_| vec![1.0]).collect())
1548        }
1549        fn supports_model(&self, _model: EmbeddingModel) -> bool {
1550            true
1551        }
1552        fn name(&self) -> &'static str {
1553            "custom-waiting"
1554        }
1555    }
1556
1557    #[tokio::test(start_paused = true)]
1558    async fn runtime_custom_inference_is_not_reported_as_builtin_admission_timeout() {
1559        let inner = Arc::new(CustomWaitingService {
1560            release: Notify::new(),
1561        });
1562        let runtime = queue_runtime(inner.clone());
1563        let mut call = Box::pin(khive_storage::scope_request_read_deadline(
1564            Duration::from_millis(100),
1565            runtime.embed_with_model("queue-test", "custom"),
1566        ));
1567        assert!(poll_once(call.as_mut()).await.is_pending());
1568        tokio::time::advance(Duration::from_millis(100)).await;
1569        let after_deadline = poll_once(call.as_mut()).await;
1570        inner.release.notify_one();
1571        assert!(
1572            after_deadline.is_pending(),
1573            "custom inference did not enter the built-in pre-admission wait: {after_deadline:?}"
1574        );
1575        assert_eq!(call.await.unwrap(), vec![1.0]);
1576    }
1577
1578    #[tokio::test]
1579    async fn blocking_adapter_rejects_oversized_job_before_enqueue() {
1580        let service = BlockingEmbeddingService::new(Arc::new(ConstVecService { dims: 1 }));
1581        let oversized =
1582            "x".repeat(lattice_embed::DEFAULT_MAX_BATCH_SIZE * lattice_embed::MAX_TEXT_BYTES + 1);
1583
1584        let error = service
1585            .embed(&[oversized], EmbeddingModel::default())
1586            .await
1587            .expect_err("an oversized embedding job must be rejected");
1588
1589        assert!(
1590            error.to_string().contains("embedding job input"),
1591            "oversized admission must use the embedding error path: {error}"
1592        );
1593        assert!(
1594            service.worker.get().is_none(),
1595            "oversized work must be rejected before the worker queue is initialized"
1596        );
1597    }
1598
1599    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1600    async fn blocking_adapter_byte_budget_rejects_excess_and_queued_jobs_complete() {
1601        const ADMITTED_JOBS: usize = 4;
1602        let byte_budget = ADMITTED_JOBS * EMBEDDING_MAX_JOB_BYTES;
1603        let inner = Arc::new(BlockingTestService::new());
1604        let service = Arc::new(BlockingEmbeddingService::with_byte_budget(
1605            Arc::clone(&inner),
1606            byte_budget,
1607        ));
1608        let max_texts = Arc::new(vec![
1609            "x".repeat(lattice_embed::MAX_TEXT_BYTES);
1610            lattice_embed::DEFAULT_MAX_BATCH_SIZE
1611        ]);
1612        let mut admitted = Vec::with_capacity(ADMITTED_JOBS);
1613
1614        for _ in 0..ADMITTED_JOBS {
1615            let service = Arc::clone(&service);
1616            let texts = Arc::clone(&max_texts);
1617            admitted.push(tokio::spawn(async move {
1618                service.embed(&texts, EmbeddingModel::default()).await
1619            }));
1620        }
1621        tokio::time::timeout(Duration::from_secs(1), async {
1622            while service.in_flight_bytes.load(Ordering::Acquire) < byte_budget {
1623                tokio::task::yield_now().await;
1624            }
1625        })
1626        .await
1627        .expect("all jobs within the byte budget must be admitted");
1628
1629        let overflow = tokio::time::timeout(
1630            Duration::from_millis(100),
1631            service.embed(&max_texts, EmbeddingModel::default()),
1632        )
1633        .await
1634        .expect("a byte-budget overflow must fail without waiting")
1635        .expect_err("a byte-budget overflow must return an embedding error");
1636
1637        inner.release();
1638        for call in admitted {
1639            call.await
1640                .expect("admitted embedding task must not panic")
1641                .expect("admitted embedding job must complete");
1642        }
1643        assert!(
1644            overflow.to_string().contains("byte budget"),
1645            "byte saturation must use the embedding failure path: {overflow}"
1646        );
1647    }
1648
1649    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1650    async fn blocking_adapter_releases_byte_budget_after_completion_and_skipped_job() {
1651        let byte_budget = "first".len() + "abandoned".len();
1652        let inner = Arc::new(BlockingTestService::new());
1653        let service = Arc::new(BlockingEmbeddingService::with_byte_budget(
1654            Arc::clone(&inner),
1655            byte_budget,
1656        ));
1657
1658        let first_service = Arc::clone(&service);
1659        let first = tokio::spawn(async move {
1660            first_service
1661                .embed(&["first".to_owned()], EmbeddingModel::default())
1662                .await
1663        });
1664        tokio::time::timeout(Duration::from_secs(1), async {
1665            while inner.entered.load(Ordering::Acquire) == 0 {
1666                tokio::task::yield_now().await;
1667            }
1668        })
1669        .await
1670        .expect("first embedding call must occupy the native worker");
1671
1672        let abandoned = tokio::time::timeout(
1673            Duration::from_millis(50),
1674            service.embed(&["abandoned".to_owned()], EmbeddingModel::default()),
1675        )
1676        .await;
1677        assert!(abandoned.is_err(), "queued embedding call must time out");
1678        assert_eq!(
1679            service.in_flight_bytes.load(Ordering::Acquire),
1680            byte_budget,
1681            "running and queued jobs must both consume the byte budget"
1682        );
1683
1684        inner.release();
1685        first
1686            .await
1687            .expect("first embedding task must not panic")
1688            .expect("first embedding call must succeed");
1689        tokio::time::timeout(Duration::from_secs(1), async {
1690            while service.in_flight_bytes.load(Ordering::Acquire) != 0 {
1691                tokio::task::yield_now().await;
1692            }
1693        })
1694        .await
1695        .expect("completed and skipped jobs must release their byte reservations");
1696
1697        let later = service
1698            .embed(&["later".to_owned()], EmbeddingModel::default())
1699            .await
1700            .expect("a later call must succeed after the byte budget is released");
1701        assert_eq!(later, vec![vec![1.0]]);
1702    }
1703
1704    #[test]
1705    fn register_and_get_provider_round_trip() {
1706        let mut reg = EmbedderRegistry::new();
1707        reg.register(ConstVecProvider::new("mock-384", 384))
1708            .expect("first registration should succeed");
1709
1710        assert!(reg.contains("mock-384"), "registered name must be present");
1711        let provider = reg.get_provider("mock-384").expect("provider must exist");
1712        assert_eq!(provider.name(), "mock-384");
1713        assert_eq!(provider.dimensions(), 384);
1714    }
1715
1716    #[tokio::test]
1717    async fn duplicate_registration_preserves_serving_provider() {
1718        let mut reg = EmbedderRegistry::new();
1719        reg.register(ConstVecProvider::new("shared", 128))
1720            .expect("first registration should succeed");
1721        reg.register(ConstVecProvider::new("other", 64))
1722            .expect("unique registration should succeed");
1723        reg.get_service("shared")
1724            .await
1725            .expect("serving provider should build");
1726        let error = reg
1727            .register(ConstVecProvider::new("shared", 256))
1728            .expect_err("duplicate registration should fail");
1729        assert!(
1730            matches!(error, RuntimeError::InvalidInput(message) if message.contains("shared") && message.contains("serving"))
1731        );
1732
1733        let provider = reg.get_provider("shared").expect("provider must exist");
1734        assert_eq!(provider.dimensions(), 128);
1735        assert_eq!(reg.names(), vec!["shared".to_owned(), "other".to_owned()]);
1736    }
1737
1738    #[tokio::test]
1739    async fn runtime_try_register_embedder_returns_duplicate_error() {
1740        let runtime = crate::KhiveRuntime::memory().expect("memory runtime");
1741        runtime
1742            .try_register_embedder(ConstVecProvider::new("first-custom", 128))
1743            .expect("first registration should succeed");
1744        runtime
1745            .try_register_embedder(ConstVecProvider::new("second-custom", 256))
1746            .expect("second registration should succeed");
1747
1748        let expected = runtime.registered_embedding_model_names();
1749        assert!(expected.ends_with(&["first-custom".to_owned(), "second-custom".to_owned()]));
1750        for _ in 0..100 {
1751            assert_eq!(runtime.registered_embedding_model_names(), expected);
1752        }
1753        runtime
1754            .embedder("first-custom")
1755            .await
1756            .expect("first custom provider should build");
1757
1758        let error = runtime
1759            .try_register_embedder(ConstVecProvider::new("first-custom", 512))
1760            .expect_err("duplicate registration should fail");
1761        assert!(
1762            matches!(error, RuntimeError::InvalidInput(message) if message.contains("first-custom") && message.contains("serving"))
1763        );
1764    }
1765
1766    #[test]
1767    fn bound_engine_list_is_fixed_once_bound() {
1768        let engine = |name: &str, dims: Option<i64>| crate::engine_config::EngineConfig {
1769            name: name.to_owned(),
1770            weight: 1.0,
1771            dims,
1772        };
1773        let mut reg = EmbedderRegistry::new();
1774        for (name, dimensions) in [("bound-a", 8), ("bound-b", 4)] {
1775            reg.register(ConstVecProvider::new(name, dimensions))
1776                .expect("unique registration should succeed");
1777        }
1778        let bound = vec![engine("bound-a", Some(8)), engine("bound-b", None)];
1779        reg.bind_engines(&bound)
1780            .expect("first binding should succeed");
1781        reg.bind_engines(&bound)
1782            .expect("binding the same ordered list again is idempotent");
1783
1784        // Another order, a shorter list and a changed field are each a different list.
1785        for different in [
1786            vec![engine("bound-b", None), engine("bound-a", Some(8))],
1787            vec![engine("bound-a", Some(8))],
1788            vec![engine("bound-a", Some(8)), engine("bound-b", Some(4))],
1789        ] {
1790            let error = reg
1791                .bind_engines(&different)
1792                .expect_err("a bound registry must refuse a different list");
1793            assert!(
1794                matches!(&error, RuntimeError::InvalidInput(message) if message.contains("different ordered list")),
1795                "{error:?}"
1796            );
1797            assert_eq!(reg.bound_engines(), Some(bound.as_slice()));
1798        }
1799    }
1800
1801    #[test]
1802    fn names_preserve_registration_order_across_calls() {
1803        let mut reg = EmbedderRegistry::new();
1804        for (name, dimensions) in [("e", 5), ("b", 4), ("d", 3), ("a", 2), ("c", 1)] {
1805            reg.register(ConstVecProvider::new(name, dimensions))
1806                .expect("unique registration should succeed");
1807        }
1808
1809        let expected = ["e", "b", "d", "a", "c"].map(|name| name.to_owned());
1810        for _ in 0..100 {
1811            assert_eq!(reg.names(), expected);
1812        }
1813    }
1814
1815    #[test]
1816    fn names_returns_all_registered() {
1817        let mut reg = EmbedderRegistry::new();
1818        reg.register(ConstVecProvider::new("model-a", 64))
1819            .expect("unique registration should succeed");
1820        reg.register(ConstVecProvider::new("model-b", 128))
1821            .expect("unique registration should succeed");
1822        reg.register(ConstVecProvider::new("model-c", 256))
1823            .expect("unique registration should succeed");
1824
1825        let mut names = reg.names();
1826        names.sort();
1827        assert_eq!(names, vec!["model-a", "model-b", "model-c"]);
1828    }
1829
1830    #[tokio::test]
1831    async fn get_service_unknown_name_returns_error() {
1832        let reg = EmbedderRegistry::new();
1833        let result = reg.get_service("does-not-exist").await;
1834        let err = result.err().expect("expected Err for unknown name, got Ok");
1835        assert!(
1836            matches!(err, RuntimeError::UnknownModel(ref n) if n == "does-not-exist"),
1837            "expected UnknownModel, got {err:?}"
1838        );
1839    }
1840
1841    #[tokio::test]
1842    async fn get_service_calls_build_once() {
1843        let counter = Arc::new(AtomicUsize::new(0));
1844        let provider = ConstVecProvider {
1845            name: "cached-model".to_owned(),
1846            dims: 32,
1847            build_calls: Arc::clone(&counter),
1848        };
1849        let mut reg = EmbedderRegistry::new();
1850        reg.register(provider)
1851            .expect("unique registration should succeed");
1852
1853        let _ = reg.get_service("cached-model").await.unwrap();
1854        let _ = reg.get_service("cached-model").await.unwrap();
1855        let _ = reg.get_service("cached-model").await.unwrap();
1856
1857        assert_eq!(
1858            counter.load(Ordering::SeqCst),
1859            1,
1860            "build must be called exactly once regardless of get_service call count"
1861        );
1862    }
1863
1864    struct SlowBuildProvider {
1865        name: String,
1866        dims: usize,
1867        build_calls: Arc<AtomicUsize>,
1868    }
1869
1870    #[async_trait]
1871    impl EmbedderProvider for SlowBuildProvider {
1872        fn name(&self) -> &str {
1873            &self.name
1874        }
1875
1876        fn dimensions(&self) -> usize {
1877            self.dims
1878        }
1879
1880        async fn build(&self) -> RuntimeResult<Arc<dyn EmbeddingService>> {
1881            self.build_calls.fetch_add(1, Ordering::SeqCst);
1882            tokio::time::sleep(Duration::from_millis(50)).await;
1883            Ok(Arc::new(ConstVecService { dims: self.dims }))
1884        }
1885    }
1886
1887    #[tokio::test(flavor = "multi_thread", worker_threads = 8)]
1888    async fn concurrent_cold_resolutions_single_flight_one_build() {
1889        const CALLERS: usize = 16;
1890        let counter = Arc::new(AtomicUsize::new(0));
1891        let mut reg = EmbedderRegistry::new();
1892        reg.register(SlowBuildProvider {
1893            name: "cold-model".to_owned(),
1894            dims: 8,
1895            build_calls: Arc::clone(&counter),
1896        })
1897        .expect("unique registration should succeed");
1898        let reg = Arc::new(reg);
1899
1900        let mut callers = Vec::with_capacity(CALLERS);
1901        for _ in 0..CALLERS {
1902            let reg = Arc::clone(&reg);
1903            callers.push(tokio::spawn(
1904                async move { reg.get_service("cold-model").await },
1905            ));
1906        }
1907
1908        for caller in callers {
1909            let service = caller
1910                .await
1911                .expect("resolution task must not panic")
1912                .expect("every concurrent cold resolution must receive a working service");
1913            assert_eq!(service.name(), "const-vec-service");
1914        }
1915
1916        assert_eq!(
1917            counter.load(Ordering::SeqCst),
1918            1,
1919            "concurrent cold resolutions must share a single in-flight build()"
1920        );
1921    }
1922}