1use std::collections::HashMap;
8use std::sync::atomic::{AtomicBool, AtomicU8, AtomicUsize, Ordering};
9use std::sync::{Arc, Mutex, OnceLock};
10
11use async_trait::async_trait;
12use lattice_embed::{
13 CachedEmbeddingService, EmbeddingModel, EmbeddingRole, EmbeddingService,
14 NativeEmbeddingService, TokenCount, DEFAULT_MAX_BATCH_SIZE, MAX_TEXT_BYTES,
15};
16use tokio::sync::{mpsc, Notify, OnceCell};
17
18use crate::error::{RuntimeError, RuntimeResult};
19
20const ADMISSION_WAITING: u8 = 1;
21const ADMISSION_ACCEPTED: u8 = 2;
22
23struct EmbeddingAdmission {
24 deadline: khive_storage::RequestReadDeadline,
25 request: khive_storage::RequestReadContext,
26 state: AtomicU8,
27 changed: Notify,
28}
29
30impl EmbeddingAdmission {
31 fn expired(&self) -> bool {
32 tokio::time::Instant::now() >= self.deadline.async_at()
33 || self.request.stop_reason().is_some()
34 }
35}
36
37tokio::task_local! {
38 static EMBEDDING_ADMISSION: Arc<EmbeddingAdmission>;
39}
40
41pub(crate) async fn with_embedding_admission<T>(
44 future: impl std::future::Future<Output = lattice_embed::Result<T>>,
45) -> RuntimeResult<T> {
46 let deadline = khive_storage::effective_request_read_deadline(
47 khive_storage::RequestReadDeadline::after(khive_storage::request_read_timeout_from_env()),
48 );
49 let timeout = deadline
50 .async_at()
51 .saturating_duration_since(tokio::time::Instant::now());
52 let admission = Arc::new(EmbeddingAdmission {
53 deadline,
54 request: khive_storage::capture_request_read_context(),
55 state: AtomicU8::new(0),
56 changed: Notify::new(),
57 });
58 EMBEDDING_ADMISSION
59 .scope(Arc::clone(&admission), async move {
60 tokio::pin!(future);
61 let stopped = async {
62 tokio::select! {
63 _ = tokio::time::sleep_until(deadline.async_at()) => {},
64 _ = admission.request.clone().wait_for_stop() => {},
65 }
66 };
67 tokio::pin!(stopped);
68 let mut bound_expired = false;
69 loop {
70 tokio::select! {
71 biased;
73 result = &mut future => return result.map_err(RuntimeError::from),
74 _ = admission.changed.notified() => {},
75 _ = &mut stopped, if !bound_expired => bound_expired = true,
76 }
77 match admission.state.load(Ordering::Acquire) {
78 ADMISSION_ACCEPTED => {}
81 ADMISSION_WAITING if bound_expired => {
82 return Err(RuntimeError::Storage(
83 khive_storage::StorageError::AdmissionTimeout {
84 operation: "embedding admission".into(),
85 timeout_ms: u64::try_from(timeout.as_millis()).unwrap_or(u64::MAX),
86 pool_identity: None,
87 },
88 ));
89 }
90 _ => {}
93 }
94 }
95 })
96 .await
97}
98
99#[derive(Clone, Copy)]
100enum EmbeddingCall {
101 Generic,
102 Query,
103 Passage,
104 CountTokens(EmbeddingRole),
105}
106
107enum EmbeddingOutput {
108 Vectors(Vec<Vec<f32>>),
109 TokenCounts(Vec<TokenCount>),
110}
111
112const EMBEDDING_QUEUE_CAPACITY: usize = 32;
113const EMBEDDING_MAX_JOB_BYTES: usize = DEFAULT_MAX_BATCH_SIZE * MAX_TEXT_BYTES;
114const EMBEDDING_QUEUE_BYTE_BUDGET: usize = EMBEDDING_QUEUE_CAPACITY * 128 * MAX_TEXT_BYTES;
116
117struct InFlightBytes {
118 counter: Arc<AtomicUsize>,
119 bytes: usize,
120}
121
122impl InFlightBytes {
123 fn reserve(
124 counter: Arc<AtomicUsize>,
125 byte_budget: usize,
126 bytes: usize,
127 ) -> lattice_embed::Result<Self> {
128 let mut current = counter.load(Ordering::Acquire);
129 loop {
130 let Some(next) = current.checked_add(bytes) else {
131 return Err(lattice_embed::EmbedError::Internal(format!(
132 "embedding worker byte budget exceeded: in-flight byte count overflowed the {byte_budget}-byte budget"
133 )));
134 };
135 if next > byte_budget {
136 return Err(lattice_embed::EmbedError::Internal(format!(
137 "embedding worker byte budget exceeded: {current} in flight + {bytes} job bytes > {byte_budget}"
138 )));
139 }
140 match counter.compare_exchange_weak(current, next, Ordering::AcqRel, Ordering::Acquire)
141 {
142 Ok(_) => return Ok(Self { counter, bytes }),
143 Err(observed) => current = observed,
144 }
145 }
146 }
147}
148
149impl Drop for InFlightBytes {
150 fn drop(&mut self) {
151 let previous = self.counter.fetch_sub(self.bytes, Ordering::AcqRel);
152 debug_assert!(previous >= self.bytes, "embedding byte counter underflow");
153 }
154}
155
156struct EmbeddingJob {
157 texts: Vec<String>,
158 model: EmbeddingModel,
159 call: EmbeddingCall,
160 reply: tokio::sync::oneshot::Sender<lattice_embed::Result<EmbeddingOutput>>,
161 _in_flight: InFlightBytes,
162}
163
164pub(crate) struct BlockingEmbeddingService<S> {
167 inner: Arc<S>,
168 worker: OnceLock<Result<mpsc::Sender<EmbeddingJob>, String>>,
169 in_flight_bytes: Arc<AtomicUsize>,
170 byte_budget: usize,
171}
172
173impl<S> BlockingEmbeddingService<S> {
174 pub(crate) fn new(inner: Arc<S>) -> Self {
175 Self {
176 inner,
177 worker: OnceLock::new(),
178 in_flight_bytes: Arc::new(AtomicUsize::new(0)),
179 byte_budget: EMBEDDING_QUEUE_BYTE_BUDGET,
180 }
181 }
182
183 #[cfg(test)]
184 fn with_byte_budget(inner: Arc<S>, byte_budget: usize) -> Self {
185 Self {
186 inner,
187 worker: OnceLock::new(),
188 in_flight_bytes: Arc::new(AtomicUsize::new(0)),
189 byte_budget,
190 }
191 }
192}
193
194impl<S: EmbeddingService + 'static> BlockingEmbeddingService<S> {
195 fn input_bytes(texts: &[String]) -> lattice_embed::Result<usize> {
196 if texts.is_empty() {
197 return Err(lattice_embed::EmbedError::InvalidInput(
198 "no texts provided".to_owned(),
199 ));
200 }
201 let input_bytes = texts.iter().try_fold(0usize, |total, text| {
202 total.checked_add(text.len()).ok_or_else(|| {
203 lattice_embed::EmbedError::InvalidInput(format!(
204 "embedding job input exceeds the {EMBEDDING_MAX_JOB_BYTES}-byte maximum"
205 ))
206 })
207 })?;
208 if input_bytes > EMBEDDING_MAX_JOB_BYTES {
209 return Err(lattice_embed::EmbedError::InvalidInput(format!(
210 "embedding job input is {input_bytes} bytes; maximum is {EMBEDDING_MAX_JOB_BYTES} bytes"
211 )));
212 }
213 if texts.len() > DEFAULT_MAX_BATCH_SIZE {
214 return Err(lattice_embed::EmbedError::InvalidInput(format!(
215 "batch size {} exceeds maximum {DEFAULT_MAX_BATCH_SIZE}",
216 texts.len()
217 )));
218 }
219 if let Some(text) = texts.iter().find(|text| text.len() > MAX_TEXT_BYTES) {
220 return Err(lattice_embed::EmbedError::TextTooLong {
221 length: text.len(),
222 max: MAX_TEXT_BYTES,
223 });
224 }
225 Ok(input_bytes)
226 }
227
228 fn worker(&self) -> lattice_embed::Result<&mpsc::Sender<EmbeddingJob>> {
229 self.worker
230 .get_or_init(|| {
231 let (sender, receiver) = mpsc::channel(EMBEDDING_QUEUE_CAPACITY);
232 let inner = Arc::clone(&self.inner);
233 let runtime = tokio::runtime::Handle::current();
234 std::thread::Builder::new()
235 .name("khive-embedding".to_owned())
236 .spawn(move || Self::run_worker(inner, runtime, receiver))
237 .map(|_| sender)
238 .map_err(|error| error.to_string())
239 })
240 .as_ref()
241 .map_err(|error| lattice_embed::EmbedError::Internal(error.clone()))
242 }
243
244 fn run_worker(
245 inner: Arc<S>,
246 runtime: tokio::runtime::Handle,
247 mut receiver: mpsc::Receiver<EmbeddingJob>,
248 ) {
249 while let Some(job) = receiver.blocking_recv() {
250 if job.reply.is_closed() {
251 continue;
252 }
253 let result = runtime.block_on(async {
254 match job.call {
255 EmbeddingCall::Generic => inner
256 .embed(&job.texts, job.model)
257 .await
258 .map(EmbeddingOutput::Vectors),
259 EmbeddingCall::Query => inner
260 .embed_query(&job.texts, job.model)
261 .await
262 .map(EmbeddingOutput::Vectors),
263 EmbeddingCall::Passage => inner
264 .embed_passage(&job.texts, job.model)
265 .await
266 .map(EmbeddingOutput::Vectors),
267 EmbeddingCall::CountTokens(role) => inner
268 .count_tokens(&job.texts, job.model, role)
269 .await
270 .map(EmbeddingOutput::TokenCounts),
271 }
272 });
273 let _ = job.reply.send(result);
274 }
275 }
276
277 async fn run(
278 &self,
279 texts: &[String],
280 model: EmbeddingModel,
281 call: EmbeddingCall,
282 ) -> lattice_embed::Result<Vec<Vec<f32>>> {
283 match self.run_job(texts, model, call).await? {
284 EmbeddingOutput::Vectors(vectors) => Ok(vectors),
285 EmbeddingOutput::TokenCounts(_) => Err(lattice_embed::EmbedError::Internal(
286 "embedding worker returned token counts for a vector request".to_owned(),
287 )),
288 }
289 }
290
291 async fn run_job(
292 &self,
293 texts: &[String],
294 model: EmbeddingModel,
295 call: EmbeddingCall,
296 ) -> lattice_embed::Result<EmbeddingOutput> {
297 let input_bytes = Self::input_bytes(texts)?;
298 let sender = self.worker()?;
299 let admission = EMBEDDING_ADMISSION.try_with(Arc::clone).ok();
300 let permit = if let Some(admission) = &admission {
301 admission.state.store(ADMISSION_WAITING, Ordering::Release);
302 admission.changed.notify_one();
303 if admission.expired() {
304 return std::future::pending().await;
305 }
306 sender.reserve().await.map_err(|_| {
307 lattice_embed::EmbedError::Internal(
308 "embedding worker channel is disconnected".to_owned(),
309 )
310 })?
311 } else {
312 sender.try_reserve().map_err(|error| match error {
315 mpsc::error::TrySendError::Full(_) => {
316 lattice_embed::EmbedError::Internal("embedding worker queue is full".to_owned())
317 }
318 mpsc::error::TrySendError::Closed(_) => lattice_embed::EmbedError::Internal(
319 "embedding worker channel is disconnected".to_owned(),
320 ),
321 })?
322 };
323 if admission
326 .as_ref()
327 .is_some_and(|admission| admission.expired())
328 {
329 return std::future::pending().await;
330 }
331 let in_flight = InFlightBytes::reserve(
332 Arc::clone(&self.in_flight_bytes),
333 self.byte_budget,
334 input_bytes,
335 )?;
336 let (reply, receiver) = tokio::sync::oneshot::channel();
337 let job = EmbeddingJob {
338 texts: texts.to_vec(),
339 model,
340 call,
341 reply,
342 _in_flight: in_flight,
343 };
344 if let Some(admission) = &admission {
345 if admission.expired() {
346 return std::future::pending().await;
347 }
348 admission.state.store(ADMISSION_ACCEPTED, Ordering::Release);
349 admission.changed.notify_one();
350 }
351 permit.send(job);
352 receiver
353 .await
354 .map_err(|error| lattice_embed::EmbedError::Internal(error.to_string()))?
355 }
356}
357
358#[async_trait]
359impl<S: EmbeddingService + 'static> EmbeddingService for BlockingEmbeddingService<S> {
360 async fn embed(
361 &self,
362 texts: &[String],
363 model: EmbeddingModel,
364 ) -> lattice_embed::Result<Vec<Vec<f32>>> {
365 self.run(texts, model, EmbeddingCall::Generic).await
366 }
367
368 async fn embed_with_role(
369 &self,
370 texts: &[String],
371 model: EmbeddingModel,
372 role: EmbeddingRole,
373 ) -> lattice_embed::Result<Vec<Vec<f32>>> {
374 let call = match role {
377 EmbeddingRole::Generic => EmbeddingCall::Generic,
378 EmbeddingRole::Query => EmbeddingCall::Query,
379 EmbeddingRole::Passage => EmbeddingCall::Passage,
380 _ => {
381 return Err(lattice_embed::EmbedError::InvalidInput(
382 "unsupported embedding role".to_owned(),
383 ))
384 }
385 };
386 self.run(texts, model, call).await
387 }
388
389 async fn embed_query(
390 &self,
391 texts: &[String],
392 model: EmbeddingModel,
393 ) -> lattice_embed::Result<Vec<Vec<f32>>> {
394 self.run(texts, model, EmbeddingCall::Query).await
395 }
396
397 async fn embed_passage(
398 &self,
399 texts: &[String],
400 model: EmbeddingModel,
401 ) -> lattice_embed::Result<Vec<Vec<f32>>> {
402 self.run(texts, model, EmbeddingCall::Passage).await
403 }
404
405 async fn count_tokens(
406 &self,
407 texts: &[String],
408 model: EmbeddingModel,
409 role: EmbeddingRole,
410 ) -> lattice_embed::Result<Vec<TokenCount>> {
411 match self
412 .run_job(texts, model, EmbeddingCall::CountTokens(role))
413 .await?
414 {
415 EmbeddingOutput::TokenCounts(counts) => Ok(counts),
416 EmbeddingOutput::Vectors(_) => Err(lattice_embed::EmbedError::Internal(
417 "embedding worker returned vectors for a token-count request".to_owned(),
418 )),
419 }
420 }
421
422 fn model_config(&self, model: EmbeddingModel) -> lattice_embed::ModelConfig {
423 self.inner.model_config(model)
424 }
425
426 fn supports_model(&self, model: EmbeddingModel) -> bool {
427 self.inner.supports_model(model)
428 }
429
430 fn name(&self) -> &'static str {
431 self.inner.name()
432 }
433}
434
435#[async_trait]
445pub trait EmbedderProvider: Send + Sync {
446 fn name(&self) -> &str;
453
454 fn dimensions(&self) -> usize;
459
460 async fn build(&self) -> RuntimeResult<Arc<dyn EmbeddingService>>;
466}
467
468pub(crate) struct EmbedderEntry {
471 provider: Arc<dyn EmbedderProvider>,
472 cell: Arc<OnceCell<Arc<dyn EmbeddingService>>>,
473 serving: Arc<AtomicBool>,
474 audited_document_preparation: bool,
477}
478
479impl Clone for EmbedderEntry {
480 fn clone(&self) -> Self {
481 Self {
482 provider: Arc::clone(&self.provider),
483 cell: Arc::clone(&self.cell),
484 serving: Arc::clone(&self.serving),
485 audited_document_preparation: self.audited_document_preparation,
486 }
487 }
488}
489
490#[derive(Clone, Default)]
497pub struct EmbedderRegistry {
498 entries: HashMap<String, EmbedderEntry>,
499 ordered_names: Vec<String>,
500 bound_engines: Option<Vec<crate::engine_config::EngineConfig>>,
501 initialization: Arc<Mutex<()>>,
502}
503
504impl EmbedderRegistry {
505 pub fn new() -> Self {
507 Self {
508 entries: HashMap::new(),
509 ordered_names: Vec::new(),
510 bound_engines: None,
511 initialization: Arc::new(Mutex::new(())),
512 }
513 }
514
515 pub fn register<P: EmbedderProvider + 'static>(&mut self, provider: P) -> RuntimeResult<()> {
521 self.insert(provider, false)
522 }
523
524 pub(crate) fn register_builtin(&mut self, provider: LatticeEmbedderProvider) {
527 self.insert(provider, true)
528 .expect("configured built-in embedding models must have unique names");
529 }
530
531 #[cfg(feature = "test-internals")]
534 pub fn register_test_audited<P: EmbedderProvider + 'static>(
535 &mut self,
536 model: EmbeddingModel,
537 provider: P,
538 ) {
539 assert_eq!(provider.name(), model.to_string());
540 self.insert(TestAuditedProvider { provider }, true)
541 .expect("test audited provider names must be unique");
542 }
543
544 #[cfg(feature = "test-internals")]
553 pub fn replace_bound_for_test<P: EmbedderProvider + 'static>(
554 &mut self,
555 provider: P,
556 ) -> RuntimeResult<()> {
557 let name = provider.name().to_owned();
558 if !self
559 .bound_engines
560 .as_ref()
561 .is_some_and(|engines| engines.iter().any(|engine| engine.name == name))
562 {
563 return Err(RuntimeError::InvalidInput(format!(
564 "embedding provider `{name}` is not bound to configured engines"
565 )));
566 }
567 let bound = self.entries.get(&name).ok_or_else(|| {
568 RuntimeError::InvalidInput(format!("embedding provider `{name}` is not registered"))
569 })?;
570 if bound.serving.load(Ordering::Acquire) {
571 return Err(RuntimeError::InvalidInput(format!(
572 "embedding provider `{name}` is already serving"
573 )));
574 }
575 let expected = bound.provider.dimensions();
576 if provider.dimensions() != expected {
577 return Err(RuntimeError::InvalidInput(format!(
578 "embedding provider `{name}` has dimensions {}, but the bound provider has {expected}",
579 provider.dimensions()
580 )));
581 }
582 self.entries.insert(
583 name,
584 EmbedderEntry {
585 provider: Arc::new(provider),
586 cell: Arc::new(OnceCell::new()),
587 serving: Arc::new(AtomicBool::new(false)),
588 audited_document_preparation: false,
589 },
590 );
591 Ok(())
592 }
593
594 fn insert<P: EmbedderProvider + 'static>(
595 &mut self,
596 provider: P,
597 audited_document_preparation: bool,
598 ) -> RuntimeResult<()> {
599 let name = provider.name().to_owned();
600 if self
601 .bound_engines
602 .as_ref()
603 .is_some_and(|engines| engines.iter().any(|engine| engine.name == name))
604 {
605 return Err(RuntimeError::InvalidInput(format!(
606 "embedding provider `{name}` is already bound to configured engines"
607 )));
608 }
609 if self
610 .entries
611 .get(&name)
612 .is_some_and(|entry| entry.serving.load(Ordering::Acquire))
613 {
614 return Err(RuntimeError::InvalidInput(format!(
615 "embedding provider `{name}` is already serving"
616 )));
617 }
618 let entry = EmbedderEntry {
619 provider: Arc::new(provider),
620 cell: Arc::new(OnceCell::new()),
621 serving: Arc::new(AtomicBool::new(false)),
622 audited_document_preparation,
623 };
624 if self.entries.contains_key(&name) {
625 self.entries.insert(name, entry);
626 } else {
627 self.ordered_names.push(name.clone());
628 self.entries.insert(name, entry);
629 }
630 Ok(())
631 }
632
633 pub fn get_provider(&self, name: &str) -> Option<&dyn EmbedderProvider> {
635 self.entries.get(name).map(|e| e.provider.as_ref())
636 }
637
638 pub fn contains(&self, name: &str) -> bool {
640 self.entries.contains_key(name)
641 }
642
643 pub fn names(&self) -> Vec<String> {
645 self.ordered_names.clone()
646 }
647
648 pub(crate) fn initialization_lock(&self) -> Arc<Mutex<()>> {
649 Arc::clone(&self.initialization)
650 }
651
652 pub(crate) fn bound_engines(&self) -> Option<&[crate::engine_config::EngineConfig]> {
653 self.bound_engines.as_deref()
654 }
655
656 pub(crate) fn bind_engines(
657 &mut self,
658 engines: &[crate::engine_config::EngineConfig],
659 ) -> RuntimeResult<()> {
660 if let Some(bound) = &self.bound_engines {
661 return if bound == engines {
662 Ok(())
663 } else {
664 Err(RuntimeError::InvalidInput(
665 "configured embedding engines are already bound to a different ordered list"
666 .into(),
667 ))
668 };
669 }
670 for engine in engines {
671 let provider = self.get_provider(&engine.name).ok_or_else(|| {
672 RuntimeError::InvalidInput(format!(
673 "embedding provider `{}` is not registered at startup",
674 engine.name
675 ))
676 })?;
677 let actual = provider.dimensions();
678 if actual == 0 || u32::try_from(actual).is_err() {
679 return Err(RuntimeError::InvalidInput(format!(
680 "embedding provider `{}` has invalid dimensions {actual}; expected 1..=4294967295",
681 engine.name
682 )));
683 }
684 if let Some(model) = crate::config::parse_embedding_model_alias(&engine.name) {
685 if actual != model.dimensions() {
686 return Err(RuntimeError::InvalidInput(format!(
687 "embedding provider `{}` has dimensions {actual}, but its built-in storage binding requires {}",
688 engine.name,
689 model.dimensions()
690 )));
691 }
692 }
693 engine
694 .check_dimensions(actual)
695 .map_err(|error| RuntimeError::InvalidInput(error.to_string()))?;
696 }
697 self.bound_engines = Some(engines.to_vec());
698 Ok(())
699 }
700
701 pub(crate) fn get_entry(&self, name: &str) -> Option<EmbedderEntry> {
709 let entry = self.entries.get(name)?;
710 entry.serving.store(true, Ordering::Release);
711 Some(entry.clone())
712 }
713
714 pub async fn get_service(&self, name: &str) -> RuntimeResult<Arc<dyn EmbeddingService>> {
723 let entry = self
724 .entries
725 .get(name)
726 .ok_or_else(|| RuntimeError::UnknownModel(name.to_string()))?
727 .clone();
728 entry.serving.store(true, Ordering::Release);
729
730 Ok(entry.resolve().await?.0)
731 }
732}
733
734#[cfg(feature = "test-internals")]
735struct TestAuditedProvider<P> {
736 provider: P,
737}
738
739#[cfg(feature = "test-internals")]
740#[async_trait]
741impl<P: EmbedderProvider> EmbedderProvider for TestAuditedProvider<P> {
742 fn name(&self) -> &str {
743 self.provider.name()
744 }
745
746 fn dimensions(&self) -> usize {
747 self.provider.dimensions()
748 }
749
750 async fn build(&self) -> RuntimeResult<Arc<dyn EmbeddingService>> {
751 Ok(Arc::new(TestAuditedService(self.provider.build().await?)))
752 }
753}
754
755#[cfg(feature = "test-internals")]
759struct TestAuditedService(Arc<dyn EmbeddingService>);
760
761#[cfg(feature = "test-internals")]
762#[async_trait]
763impl EmbeddingService for TestAuditedService {
764 async fn embed(
765 &self,
766 texts: &[String],
767 model: EmbeddingModel,
768 ) -> lattice_embed::Result<Vec<Vec<f32>>> {
769 self.0.embed(texts, model).await
770 }
771
772 fn supports_model(&self, model: EmbeddingModel) -> bool {
773 self.0.supports_model(model)
774 }
775
776 fn name(&self) -> &'static str {
777 self.0.name()
778 }
779}
780
781impl EmbedderEntry {
782 pub(crate) fn cached_service(&self) -> Option<Arc<dyn EmbeddingService>> {
783 self.cell.get().map(Arc::clone)
784 }
785
786 pub(crate) fn has_audited_document_preparation(&self) -> bool {
787 self.audited_document_preparation
788 }
789
790 pub(crate) async fn resolve(self) -> RuntimeResult<(Arc<dyn EmbeddingService>, Option<i64>)> {
801 let mut own_init_duration_us: Option<i64> = None;
802 let provider = Arc::clone(&self.provider);
803 let init_duration_us = &mut own_init_duration_us;
804 let svc = self
805 .cell
806 .get_or_try_init(|| async move {
807 let init_start = std::time::Instant::now();
808 let svc = provider.build().await.map_err(|e| {
809 crate::error::RuntimeError::Internal(format!(
810 "EmbedderProvider '{}' build() failed: {e}",
811 provider.name()
812 ))
813 })?;
814 *init_duration_us = Some(init_start.elapsed().as_micros() as i64);
815 Ok::<_, RuntimeError>(svc)
816 })
817 .await?;
818 Ok((Arc::clone(svc), own_init_duration_us))
819 }
820}
821
822pub struct LatticeEmbedderProvider {
832 model: EmbeddingModel,
833 name: String,
835}
836
837impl LatticeEmbedderProvider {
838 pub fn new(model: EmbeddingModel) -> Self {
840 let name = model.to_string();
841 Self { model, name }
842 }
843}
844
845#[async_trait]
846impl EmbedderProvider for LatticeEmbedderProvider {
847 fn name(&self) -> &str {
848 &self.name
849 }
850
851 fn dimensions(&self) -> usize {
852 self.model.dimensions()
853 }
854
855 async fn build(&self) -> RuntimeResult<Arc<dyn EmbeddingService>> {
856 let native = Arc::new(NativeEmbeddingService::with_model(self.model));
857 native.ensure_loaded().await?;
858 #[cfg(all(target_os = "linux", target_env = "gnu"))]
860 unsafe {
862 libc::malloc_trim(0);
863 }
864 Ok(cached_blocking_service(native))
865 }
866}
867
868fn cached_blocking_service<S: EmbeddingService + 'static>(
870 inner: Arc<S>,
871) -> Arc<dyn EmbeddingService> {
872 let blocking = Arc::new(BlockingEmbeddingService::new(inner));
873 Arc::new(CachedEmbeddingService::with_default_cache(blocking))
874}
875
876#[cfg(test)]
877#[path = "embedding_cache_admission_tests.rs"]
878mod cache_admission_tests;
879
880#[cfg(test)]
881#[path = "embedding_token_count_tests.rs"]
882mod token_count_tests;
883
884#[cfg(test)]
887mod tests {
888 use super::*;
889 use std::collections::HashSet;
890 use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
891 use std::sync::{Condvar, Mutex};
892 use std::time::Duration;
893 use tokio::sync::Notify;
894
895 struct ConstVecProvider {
896 name: String,
897 dims: usize,
898 build_calls: Arc<AtomicUsize>,
899 }
900
901 impl ConstVecProvider {
902 fn new(name: &str, dims: usize) -> Self {
903 Self {
904 name: name.to_owned(),
905 dims,
906 build_calls: Arc::new(AtomicUsize::new(0)),
907 }
908 }
909 }
910
911 pub(super) struct ConstVecService {
915 pub(super) dims: usize,
916 }
917
918 #[async_trait]
919 impl EmbeddingService for ConstVecService {
920 async fn embed(
921 &self,
922 texts: &[String],
923 _model: EmbeddingModel,
924 ) -> std::result::Result<Vec<Vec<f32>>, lattice_embed::EmbedError> {
925 Ok(texts.iter().map(|_| vec![1.0_f32; self.dims]).collect())
926 }
927
928 fn supports_model(&self, _model: EmbeddingModel) -> bool {
929 true
930 }
931
932 fn name(&self) -> &'static str {
933 "const-vec-service"
934 }
935 }
936
937 #[async_trait]
938 impl EmbedderProvider for ConstVecProvider {
939 fn name(&self) -> &str {
940 &self.name
941 }
942
943 fn dimensions(&self) -> usize {
944 self.dims
945 }
946
947 async fn build(&self) -> RuntimeResult<Arc<dyn EmbeddingService>> {
948 self.build_calls.fetch_add(1, Ordering::SeqCst);
949 Ok(Arc::new(ConstVecService { dims: self.dims }))
950 }
951 }
952
953 #[test]
954 fn builtin_provider_can_be_replaced_before_resolution_without_reordering() {
955 let model = EmbeddingModel::MultilingualE5Small;
956 let name = model.to_string();
957 let mut registry = EmbedderRegistry::new();
958 registry.register_builtin(LatticeEmbedderProvider::new(model));
959 let original_names = registry.names();
960
961 registry
962 .register(ConstVecProvider::new(&name, model.dimensions()))
963 .expect("pre-serving replacement should succeed");
964 assert_eq!(registry.names(), original_names);
965 assert!(!registry
966 .get_entry(&name)
967 .expect("replacement entry")
968 .has_audited_document_preparation());
969 }
970
971 struct FirstLoadBlockingService {
972 loaded: AtomicBool,
973 }
974
975 #[async_trait]
976 impl EmbeddingService for FirstLoadBlockingService {
977 async fn embed(
978 &self,
979 texts: &[String],
980 _model: EmbeddingModel,
981 ) -> lattice_embed::Result<Vec<Vec<f32>>> {
982 if !self.loaded.swap(true, Ordering::SeqCst) {
983 tokio::task::spawn_blocking(|| {})
984 .await
985 .map_err(|error| lattice_embed::EmbedError::Internal(error.to_string()))?;
986 }
987 Ok(texts.iter().map(|_| vec![1.0]).collect())
988 }
989
990 fn supports_model(&self, _model: EmbeddingModel) -> bool {
991 true
992 }
993
994 fn name(&self) -> &'static str {
995 "first-load-blocking-service"
996 }
997 }
998
999 pub(super) struct BlockingTestService {
1000 pub(super) calls: Mutex<Vec<String>>,
1001 pub(super) entered: AtomicUsize,
1002 release: (Mutex<bool>, Condvar),
1003 pub(super) thread_ids: Mutex<HashSet<std::thread::ThreadId>>,
1004 }
1005
1006 impl BlockingTestService {
1007 pub(super) fn new() -> Self {
1008 Self {
1009 calls: Mutex::new(Vec::new()),
1010 entered: AtomicUsize::new(0),
1011 release: (Mutex::new(false), Condvar::new()),
1012 thread_ids: Mutex::new(HashSet::new()),
1013 }
1014 }
1015
1016 pub(super) fn release(&self) {
1017 *self
1018 .release
1019 .0
1020 .lock()
1021 .expect("release lock must not be poisoned") = true;
1022 self.release.1.notify_all();
1023 }
1024 }
1025
1026 #[async_trait]
1027 impl EmbeddingService for BlockingTestService {
1028 async fn embed(
1029 &self,
1030 texts: &[String],
1031 _model: EmbeddingModel,
1032 ) -> lattice_embed::Result<Vec<Vec<f32>>> {
1033 let text = texts.first().cloned().unwrap_or_default();
1034 self.thread_ids
1035 .lock()
1036 .expect("thread id lock must not be poisoned")
1037 .insert(std::thread::current().id());
1038 self.calls
1039 .lock()
1040 .expect("call lock must not be poisoned")
1041 .push(text.clone());
1042 self.entered.fetch_add(1, Ordering::Release);
1043
1044 if text != "later" {
1045 let (released, wake) = &self.release;
1046 let guard = released.lock().expect("release lock must not be poisoned");
1047 let _guard = wake
1048 .wait_while(guard, |released| !*released)
1049 .expect("release lock must not be poisoned");
1050 }
1051
1052 Ok(texts.iter().map(|_| vec![1.0]).collect())
1053 }
1054
1055 async fn embed_query(
1056 &self,
1057 texts: &[String],
1058 model: EmbeddingModel,
1059 ) -> lattice_embed::Result<Vec<Vec<f32>>> {
1060 self.embed(texts, model).await
1063 }
1064
1065 fn supports_model(&self, _model: EmbeddingModel) -> bool {
1066 true
1067 }
1068
1069 fn name(&self) -> &'static str {
1070 "blocking-test-service"
1071 }
1072 }
1073
1074 #[test]
1075 fn blocking_adapter_first_use_completes_with_single_blocking_thread() {
1076 let runtime = tokio::runtime::Builder::new_current_thread()
1077 .enable_time()
1078 .max_blocking_threads(1)
1079 .build()
1080 .expect("current-thread runtime must build");
1081 let service = BlockingEmbeddingService::new(Arc::new(FirstLoadBlockingService {
1082 loaded: AtomicBool::new(false),
1083 }));
1084
1085 let result = runtime.block_on(async {
1086 tokio::time::timeout(
1087 Duration::from_secs(5),
1088 service.embed(&["first use".to_owned()], EmbeddingModel::default()),
1089 )
1090 .await
1091 });
1092 runtime.shutdown_timeout(Duration::from_secs(1));
1093
1094 let embeddings = result
1095 .expect("first-use embedding must not exhaust the blocking pool")
1096 .expect("first-use embedding must succeed");
1097 assert_eq!(embeddings, vec![vec![1.0]]);
1098 }
1099
1100 #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
1101 async fn blocking_adapter_uses_one_worker_for_concurrent_calls() {
1102 const CALL_COUNT: usize = 32;
1103 let inner = Arc::new(BlockingTestService::new());
1104 let service = Arc::new(BlockingEmbeddingService::new(Arc::clone(&inner)));
1105 let mut calls = Vec::with_capacity(CALL_COUNT);
1106
1107 for index in 0..CALL_COUNT {
1108 let service = Arc::clone(&service);
1109 calls.push(tokio::spawn(async move {
1110 service
1111 .embed(&[format!("request-{index}")], EmbeddingModel::default())
1112 .await
1113 }));
1114 }
1115
1116 let _ = tokio::time::timeout(Duration::from_millis(250), async {
1117 while inner.entered.load(Ordering::Acquire) < 2 {
1118 tokio::task::yield_now().await;
1119 }
1120 })
1121 .await;
1122 inner.release();
1123
1124 for call in calls {
1125 call.await
1126 .expect("embedding task must not panic")
1127 .expect("embedding call must succeed");
1128 }
1129 assert_eq!(
1130 inner
1131 .thread_ids
1132 .lock()
1133 .expect("thread id lock must not be poisoned")
1134 .len(),
1135 1,
1136 "concurrent calls must share one native worker thread"
1137 );
1138 }
1139
1140 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1141 async fn blocking_adapter_skips_timed_out_call_and_serves_later_call() {
1142 let inner = Arc::new(BlockingTestService::new());
1143 let service = Arc::new(BlockingEmbeddingService::new(Arc::clone(&inner)));
1144
1145 let first_service = Arc::clone(&service);
1146 let first = tokio::spawn(async move {
1147 first_service
1148 .embed(&["first".to_owned()], EmbeddingModel::default())
1149 .await
1150 });
1151 tokio::time::timeout(Duration::from_secs(1), async {
1152 while inner.entered.load(Ordering::Acquire) == 0 {
1153 tokio::task::yield_now().await;
1154 }
1155 })
1156 .await
1157 .expect("first embedding call must enter the native service");
1158
1159 let abandoned = tokio::time::timeout(
1160 Duration::from_millis(50),
1161 service.embed(&["abandoned".to_owned()], EmbeddingModel::default()),
1162 )
1163 .await;
1164 let later_service = Arc::clone(&service);
1165 let later = tokio::spawn(async move {
1166 later_service
1167 .embed(&["later".to_owned()], EmbeddingModel::default())
1168 .await
1169 });
1170 inner.release();
1171
1172 first
1173 .await
1174 .expect("first embedding task must not panic")
1175 .expect("first embedding call must succeed");
1176 let later_result = tokio::time::timeout(Duration::from_secs(1), later)
1177 .await
1178 .expect("later embedding call must be served")
1179 .expect("later embedding task must not panic")
1180 .expect("later embedding call must succeed");
1181
1182 assert!(abandoned.is_err(), "queued embedding call must time out");
1183 assert_eq!(later_result, vec![vec![1.0]]);
1184 assert_eq!(
1185 *inner.calls.lock().expect("call lock must not be poisoned"),
1186 vec!["first".to_owned(), "later".to_owned()],
1187 "the worker must skip a queued call whose receiver is closed"
1188 );
1189 }
1190
1191 pub(super) struct ReleaseWorkerOnDrop(pub(super) Arc<BlockingTestService>);
1192
1193 impl Drop for ReleaseWorkerOnDrop {
1194 fn drop(&mut self) {
1195 self.0.release();
1196 }
1197 }
1198
1199 struct ServiceProvider(Arc<dyn EmbeddingService>);
1200
1201 #[async_trait]
1202 impl EmbedderProvider for ServiceProvider {
1203 fn name(&self) -> &str {
1204 "queue-test"
1205 }
1206 fn dimensions(&self) -> usize {
1207 1
1208 }
1209 async fn build(&self) -> RuntimeResult<Arc<dyn EmbeddingService>> {
1210 Ok(Arc::clone(&self.0))
1211 }
1212 }
1213
1214 pub(super) fn queue_runtime(service: Arc<dyn EmbeddingService>) -> crate::KhiveRuntime {
1215 let runtime = crate::KhiveRuntime::memory().expect("memory runtime");
1216 runtime.register_embedder(ServiceProvider(service));
1217 runtime
1218 }
1219
1220 pub(super) async fn poll_once<F: std::future::Future + ?Sized>(
1221 mut future: std::pin::Pin<&mut F>,
1222 ) -> std::task::Poll<F::Output> {
1223 std::future::poll_fn(|cx| std::task::Poll::Ready(future.as_mut().poll(cx))).await
1224 }
1225
1226 pub(super) async fn wait_for_entered(inner: &BlockingTestService, count: usize) {
1227 let watchdog = std::time::Instant::now() + Duration::from_secs(2);
1228 while inner.entered.load(Ordering::Acquire) < count {
1229 assert!(
1230 std::time::Instant::now() < watchdog,
1231 "setup watchdog: native worker did not enter"
1232 );
1233 tokio::task::yield_now().await;
1234 }
1235 }
1236
1237 async fn drive_until_entered<F: std::future::Future + ?Sized>(
1238 mut future: std::pin::Pin<&mut F>,
1239 inner: &BlockingTestService,
1240 count: usize,
1241 ) {
1242 let watchdog = std::time::Instant::now() + Duration::from_secs(2);
1243 while inner.entered.load(Ordering::Acquire) < count {
1244 assert!(
1245 std::time::Instant::now() < watchdog,
1246 "setup watchdog: driven runtime call did not reach the native worker"
1247 );
1248 assert!(
1249 poll_once(future.as_mut()).await.is_pending(),
1250 "held inference must remain pending while the runtime call is driven"
1251 );
1252 tokio::task::yield_now().await;
1253 }
1254 }
1255
1256 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1257 async fn runtime_embedding_waits_for_capacity_and_drains_n_plus_one_calls() {
1258 let inner = Arc::new(BlockingTestService::new());
1259 let _release = ReleaseWorkerOnDrop(Arc::clone(&inner));
1260 let service = Arc::new(BlockingEmbeddingService::new(Arc::clone(&inner)));
1261 let runtime = queue_runtime(service.clone());
1262 runtime
1263 .embedder("queue-test")
1264 .await
1265 .expect("resolve provider before the admission fixture");
1266 let first_text = vec!["first".to_owned()];
1267 let mut first = Box::pin(runtime.embed_batch_with_model("queue-test", &first_text));
1268 assert!(poll_once(first.as_mut()).await.is_pending());
1269 drive_until_entered(first.as_mut(), &inner, 1).await;
1270
1271 let texts: Vec<_> = (0..EMBEDDING_QUEUE_CAPACITY)
1272 .map(|index| vec![format!("queued-{index}")])
1273 .collect();
1274 let mut queued = Vec::new();
1275 for text in &texts {
1276 let mut call = Box::pin(runtime.embed_batch_with_model("queue-test", text));
1277 assert!(
1278 poll_once(call.as_mut()).await.is_pending(),
1279 "the bounded queue must admit its N actual runtime calls"
1280 );
1281 queued.push(call);
1282 }
1283 assert_eq!(
1284 service.in_flight_bytes.load(Ordering::Acquire),
1285 first_text.iter().map(String::len).sum::<usize>()
1286 + texts.iter().flatten().map(String::len).sum::<usize>(),
1287 "the held worker and N queued runtime jobs must own their exact input bytes"
1288 );
1289 let overflow_text = vec!["overflow".to_owned()];
1290 let mut overflow = Box::pin(runtime.embed_batch_with_model("queue-test", &overflow_text));
1291 let overflow_before_release = poll_once(overflow.as_mut()).await;
1292 inner.release();
1293 assert!(
1294 overflow_before_release.is_pending(),
1295 "request N+1 must wait for a slot instead of failing: {overflow_before_release:?}"
1296 );
1297 assert_eq!(first.await.unwrap(), vec![vec![1.0]]);
1298 for call in queued {
1299 assert_eq!(call.await.unwrap(), vec![vec![1.0]]);
1300 }
1301 assert_eq!(overflow.await.unwrap(), vec![vec![1.0]]);
1302 assert_eq!(
1303 inner.entered.load(Ordering::Acquire),
1304 EMBEDDING_QUEUE_CAPACITY + 2
1305 );
1306 assert_eq!(inner.thread_ids.lock().unwrap().len(), 1);
1307 }
1308
1309 #[tokio::test(start_paused = true)]
1310 async fn runtime_embedding_expired_admission_is_retryable_and_never_enqueued() {
1311 let inner = Arc::new(BlockingTestService::new());
1312 let _release = ReleaseWorkerOnDrop(Arc::clone(&inner));
1313 let service = Arc::new(BlockingEmbeddingService::new(Arc::clone(&inner)));
1314 let runtime = queue_runtime(service.clone());
1315 runtime
1316 .embedder("queue-test")
1317 .await
1318 .expect("resolve provider before the admission fixture");
1319 let first_text = vec!["first".to_owned()];
1320 let mut first = Box::pin(runtime.embed_batch_with_model("queue-test", &first_text));
1321 assert!(poll_once(first.as_mut()).await.is_pending());
1322 drive_until_entered(first.as_mut(), &inner, 1).await;
1323 let texts: Vec<_> = (0..EMBEDDING_QUEUE_CAPACITY)
1324 .map(|index| vec![format!("queued-{index}")])
1325 .collect();
1326 let mut queued = Vec::new();
1327 for text in &texts {
1328 let mut call = Box::pin(runtime.embed_batch_with_model("queue-test", text));
1329 assert!(poll_once(call.as_mut()).await.is_pending());
1330 queued.push(call);
1331 }
1332 assert_eq!(
1333 service.in_flight_bytes.load(Ordering::Acquire),
1334 first_text.iter().map(String::len).sum::<usize>()
1335 + texts.iter().flatten().map(String::len).sum::<usize>(),
1336 "the held worker and N queued runtime jobs must own their exact input bytes"
1337 );
1338 let deadline = khive_storage::RequestReadDeadline::after(Duration::from_millis(100));
1339 let mut expired = Box::pin(khive_storage::scope_request_read_deadline_at(
1340 deadline,
1341 runtime.embed_with_model("queue-test", "expired"),
1342 ));
1343 assert!(
1344 poll_once(expired.as_mut()).await.is_pending(),
1345 "an unexpired full-queue call must wait"
1346 );
1347 tokio::time::advance(Duration::from_millis(100)).await;
1348 let result = poll_once(expired.as_mut()).await;
1349 inner.release();
1350 let error = match result {
1351 std::task::Poll::Ready(Err(error)) => error,
1352 other => panic!("expired admission must produce a typed runtime error: {other:?}"),
1353 };
1354 assert!(
1355 matches!(&error, RuntimeError::Storage(khive_storage::StorageError::AdmissionTimeout {
1356 operation, timeout_ms: 100, pool_identity: None,
1357 }) if operation == "embedding admission"),
1358 "{error:?}"
1359 );
1360 assert!(
1361 error.retryable_failure_context().is_some(),
1362 "pre-admission expiry must use the existing retryable classification"
1363 );
1364 let projected = crate::error_projection::runtime_error_value(
1365 error,
1366 crate::DomainDisposition::NotCommitted,
1367 );
1368 assert_eq!(projected["retryable"], true);
1369 assert_eq!(projected["operation"], "embedding admission");
1370 assert_eq!(first.await.unwrap(), vec![vec![1.0]]);
1371 for call in queued {
1372 assert_eq!(call.await.unwrap(), vec![vec![1.0]]);
1373 }
1374 assert!(
1375 !inner
1376 .calls
1377 .lock()
1378 .unwrap()
1379 .iter()
1380 .any(|text| text == "expired"),
1381 "an expired waiter must never reach inference"
1382 );
1383 }
1384
1385 #[tokio::test(start_paused = true)]
1386 async fn runtime_embedding_earlier_absolute_bound_is_not_renewed_when_slot_is_ready() {
1387 let inner = Arc::new(BlockingTestService::new());
1388 let _release = ReleaseWorkerOnDrop(Arc::clone(&inner));
1389 let runtime = queue_runtime(Arc::new(BlockingEmbeddingService::new(Arc::clone(&inner))));
1390 runtime
1391 .embedder("queue-test")
1392 .await
1393 .expect("resolve provider before the admission deadline");
1394 let deadline = khive_storage::RequestReadDeadline::after(Duration::from_millis(100));
1395 tokio::time::advance(Duration::from_millis(100)).await;
1396 let mut call = Box::pin(khive_storage::scope_request_read_deadline_at(
1397 deadline,
1398 runtime.embed_with_model("queue-test", "expired-ready-slot"),
1399 ));
1400 let result = poll_once(call.as_mut()).await;
1401 inner.release();
1402 assert!(
1403 matches!(
1404 result,
1405 std::task::Poll::Ready(Err(RuntimeError::Storage(
1406 khive_storage::StorageError::AdmissionTimeout { timeout_ms: 0, .. }
1407 )))
1408 ),
1409 "an already-expired original bound must refuse even an available slot: {result:?}"
1410 );
1411 assert_eq!(
1412 inner.entered.load(Ordering::Acquire),
1413 0,
1414 "capacity becoming ready must not bypass expiry"
1415 );
1416 }
1417
1418 #[tokio::test(start_paused = true)]
1419 async fn runtime_embedding_all_six_call_families_refuse_expired_admission() {
1420 let inner = Arc::new(BlockingTestService::new());
1421 let _release = ReleaseWorkerOnDrop(Arc::clone(&inner));
1422 let runtime = queue_runtime(Arc::new(BlockingEmbeddingService::new(Arc::clone(&inner))));
1423 let texts = vec!["later".to_owned()];
1424 for family in 0..6 {
1425 let result = khive_storage::scope_request_read_deadline(Duration::ZERO, async {
1426 match family {
1427 0 => runtime.embed_with_model("queue-test", "later").await,
1428 1 => runtime
1429 .embed_document_with_model_outcome("queue-test", "later")
1430 .await
1431 .map(|outcome| outcome.vector),
1432 2 => runtime.embed_query_with_model("queue-test", "later").await,
1433 3 => runtime
1434 .embed_batch_with_model("queue-test", &texts)
1435 .await
1436 .map(|vectors| vectors[0].clone()),
1437 4 => runtime
1438 .embed_document_batch_with_model("queue-test", &texts)
1439 .await
1440 .map(|vectors| vectors[0].clone()),
1441 _ => runtime
1442 .embed_query_batch_with_model("queue-test", &texts)
1443 .await
1444 .map(|vectors| vectors[0].clone()),
1445 }
1446 })
1447 .await;
1448 assert!(
1449 matches!(
1450 result,
1451 Err(RuntimeError::Storage(
1452 khive_storage::StorageError::AdmissionTimeout { .. }
1453 ))
1454 ),
1455 "family {family} must reach the actual runtime admission boundary: {result:?}"
1456 );
1457 }
1458 assert_eq!(inner.entered.load(Ordering::Acquire), 0);
1459 }
1460
1461 #[tokio::test(start_paused = true)]
1462 async fn runtime_embedding_admitted_inference_is_not_reported_as_admission_timeout() {
1463 let inner = Arc::new(BlockingTestService::new());
1464 let _release = ReleaseWorkerOnDrop(Arc::clone(&inner));
1465 let runtime = queue_runtime(Arc::new(BlockingEmbeddingService::new(Arc::clone(&inner))));
1466 runtime
1467 .embedder("queue-test")
1468 .await
1469 .expect("resolve provider before the admission fixture");
1470 let mut call = Box::pin(khive_storage::scope_request_read_deadline(
1471 Duration::from_millis(100),
1472 runtime.embed_with_model("queue-test", "first"),
1473 ));
1474 assert!(poll_once(call.as_mut()).await.is_pending());
1475 drive_until_entered(call.as_mut(), &inner, 1).await;
1476 tokio::time::advance(Duration::from_millis(100)).await;
1477 let after_deadline = poll_once(call.as_mut()).await;
1478 inner.release();
1479 assert!(after_deadline.is_pending(),
1480 "already-running inference cannot be called a pre-admission refusal: {after_deadline:?}");
1481 assert_eq!(call.await.unwrap(), vec![1.0]);
1482 }
1483
1484 #[tokio::test(start_paused = true)]
1485 async fn runtime_embedding_second_job_keeps_the_original_admission_bound() {
1486 let inner = Arc::new(BlockingTestService::new());
1487 let _release = ReleaseWorkerOnDrop(Arc::clone(&inner));
1488 let service = BlockingEmbeddingService::new(Arc::clone(&inner));
1489 let first = vec!["first".to_owned()];
1490 let second = vec!["second".to_owned()];
1491 let mut call = Box::pin(khive_storage::scope_request_read_deadline(
1492 Duration::from_millis(100),
1493 with_embedding_admission(async {
1494 service.embed(&first, EmbeddingModel::default()).await?;
1495 service.embed(&second, EmbeddingModel::default()).await
1496 }),
1497 ));
1498 assert!(poll_once(call.as_mut()).await.is_pending());
1499 drive_until_entered(call.as_mut(), &inner, 1).await;
1500 tokio::time::advance(Duration::from_millis(100)).await;
1501 assert!(
1502 poll_once(call.as_mut()).await.is_pending(),
1503 "the admitted first job must finish despite expiry"
1504 );
1505 inner.release();
1506
1507 let watchdog = std::time::Instant::now() + Duration::from_secs(2);
1509 let result = loop {
1510 if let std::task::Poll::Ready(result) = poll_once(call.as_mut()).await {
1511 break result;
1512 }
1513 assert!(
1514 std::time::Instant::now() < watchdog,
1515 "second-job admission must settle instead of waiting forever"
1516 );
1517 tokio::task::yield_now().await;
1518 };
1519 assert!(
1520 matches!(
1521 result,
1522 Err(RuntimeError::Storage(
1523 khive_storage::StorageError::AdmissionTimeout {
1524 timeout_ms: 100,
1525 ..
1526 }
1527 ))
1528 ),
1529 "the second job must use the original bound: {result:?}"
1530 );
1531 assert_eq!(*inner.calls.lock().unwrap(), vec!["first".to_owned()]);
1532 assert_eq!(inner.entered.load(Ordering::Acquire), 1);
1533 }
1534
1535 struct CustomWaitingService {
1536 release: Notify,
1537 }
1538
1539 #[async_trait]
1540 impl EmbeddingService for CustomWaitingService {
1541 async fn embed(
1542 &self,
1543 texts: &[String],
1544 _model: EmbeddingModel,
1545 ) -> lattice_embed::Result<Vec<Vec<f32>>> {
1546 self.release.notified().await;
1547 Ok(texts.iter().map(|_| vec![1.0]).collect())
1548 }
1549 fn supports_model(&self, _model: EmbeddingModel) -> bool {
1550 true
1551 }
1552 fn name(&self) -> &'static str {
1553 "custom-waiting"
1554 }
1555 }
1556
1557 #[tokio::test(start_paused = true)]
1558 async fn runtime_custom_inference_is_not_reported_as_builtin_admission_timeout() {
1559 let inner = Arc::new(CustomWaitingService {
1560 release: Notify::new(),
1561 });
1562 let runtime = queue_runtime(inner.clone());
1563 let mut call = Box::pin(khive_storage::scope_request_read_deadline(
1564 Duration::from_millis(100),
1565 runtime.embed_with_model("queue-test", "custom"),
1566 ));
1567 assert!(poll_once(call.as_mut()).await.is_pending());
1568 tokio::time::advance(Duration::from_millis(100)).await;
1569 let after_deadline = poll_once(call.as_mut()).await;
1570 inner.release.notify_one();
1571 assert!(
1572 after_deadline.is_pending(),
1573 "custom inference did not enter the built-in pre-admission wait: {after_deadline:?}"
1574 );
1575 assert_eq!(call.await.unwrap(), vec![1.0]);
1576 }
1577
1578 #[tokio::test]
1579 async fn blocking_adapter_rejects_oversized_job_before_enqueue() {
1580 let service = BlockingEmbeddingService::new(Arc::new(ConstVecService { dims: 1 }));
1581 let oversized =
1582 "x".repeat(lattice_embed::DEFAULT_MAX_BATCH_SIZE * lattice_embed::MAX_TEXT_BYTES + 1);
1583
1584 let error = service
1585 .embed(&[oversized], EmbeddingModel::default())
1586 .await
1587 .expect_err("an oversized embedding job must be rejected");
1588
1589 assert!(
1590 error.to_string().contains("embedding job input"),
1591 "oversized admission must use the embedding error path: {error}"
1592 );
1593 assert!(
1594 service.worker.get().is_none(),
1595 "oversized work must be rejected before the worker queue is initialized"
1596 );
1597 }
1598
1599 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1600 async fn blocking_adapter_byte_budget_rejects_excess_and_queued_jobs_complete() {
1601 const ADMITTED_JOBS: usize = 4;
1602 let byte_budget = ADMITTED_JOBS * EMBEDDING_MAX_JOB_BYTES;
1603 let inner = Arc::new(BlockingTestService::new());
1604 let service = Arc::new(BlockingEmbeddingService::with_byte_budget(
1605 Arc::clone(&inner),
1606 byte_budget,
1607 ));
1608 let max_texts = Arc::new(vec![
1609 "x".repeat(lattice_embed::MAX_TEXT_BYTES);
1610 lattice_embed::DEFAULT_MAX_BATCH_SIZE
1611 ]);
1612 let mut admitted = Vec::with_capacity(ADMITTED_JOBS);
1613
1614 for _ in 0..ADMITTED_JOBS {
1615 let service = Arc::clone(&service);
1616 let texts = Arc::clone(&max_texts);
1617 admitted.push(tokio::spawn(async move {
1618 service.embed(&texts, EmbeddingModel::default()).await
1619 }));
1620 }
1621 tokio::time::timeout(Duration::from_secs(1), async {
1622 while service.in_flight_bytes.load(Ordering::Acquire) < byte_budget {
1623 tokio::task::yield_now().await;
1624 }
1625 })
1626 .await
1627 .expect("all jobs within the byte budget must be admitted");
1628
1629 let overflow = tokio::time::timeout(
1630 Duration::from_millis(100),
1631 service.embed(&max_texts, EmbeddingModel::default()),
1632 )
1633 .await
1634 .expect("a byte-budget overflow must fail without waiting")
1635 .expect_err("a byte-budget overflow must return an embedding error");
1636
1637 inner.release();
1638 for call in admitted {
1639 call.await
1640 .expect("admitted embedding task must not panic")
1641 .expect("admitted embedding job must complete");
1642 }
1643 assert!(
1644 overflow.to_string().contains("byte budget"),
1645 "byte saturation must use the embedding failure path: {overflow}"
1646 );
1647 }
1648
1649 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
1650 async fn blocking_adapter_releases_byte_budget_after_completion_and_skipped_job() {
1651 let byte_budget = "first".len() + "abandoned".len();
1652 let inner = Arc::new(BlockingTestService::new());
1653 let service = Arc::new(BlockingEmbeddingService::with_byte_budget(
1654 Arc::clone(&inner),
1655 byte_budget,
1656 ));
1657
1658 let first_service = Arc::clone(&service);
1659 let first = tokio::spawn(async move {
1660 first_service
1661 .embed(&["first".to_owned()], EmbeddingModel::default())
1662 .await
1663 });
1664 tokio::time::timeout(Duration::from_secs(1), async {
1665 while inner.entered.load(Ordering::Acquire) == 0 {
1666 tokio::task::yield_now().await;
1667 }
1668 })
1669 .await
1670 .expect("first embedding call must occupy the native worker");
1671
1672 let abandoned = tokio::time::timeout(
1673 Duration::from_millis(50),
1674 service.embed(&["abandoned".to_owned()], EmbeddingModel::default()),
1675 )
1676 .await;
1677 assert!(abandoned.is_err(), "queued embedding call must time out");
1678 assert_eq!(
1679 service.in_flight_bytes.load(Ordering::Acquire),
1680 byte_budget,
1681 "running and queued jobs must both consume the byte budget"
1682 );
1683
1684 inner.release();
1685 first
1686 .await
1687 .expect("first embedding task must not panic")
1688 .expect("first embedding call must succeed");
1689 tokio::time::timeout(Duration::from_secs(1), async {
1690 while service.in_flight_bytes.load(Ordering::Acquire) != 0 {
1691 tokio::task::yield_now().await;
1692 }
1693 })
1694 .await
1695 .expect("completed and skipped jobs must release their byte reservations");
1696
1697 let later = service
1698 .embed(&["later".to_owned()], EmbeddingModel::default())
1699 .await
1700 .expect("a later call must succeed after the byte budget is released");
1701 assert_eq!(later, vec![vec![1.0]]);
1702 }
1703
1704 #[test]
1705 fn register_and_get_provider_round_trip() {
1706 let mut reg = EmbedderRegistry::new();
1707 reg.register(ConstVecProvider::new("mock-384", 384))
1708 .expect("first registration should succeed");
1709
1710 assert!(reg.contains("mock-384"), "registered name must be present");
1711 let provider = reg.get_provider("mock-384").expect("provider must exist");
1712 assert_eq!(provider.name(), "mock-384");
1713 assert_eq!(provider.dimensions(), 384);
1714 }
1715
1716 #[tokio::test]
1717 async fn duplicate_registration_preserves_serving_provider() {
1718 let mut reg = EmbedderRegistry::new();
1719 reg.register(ConstVecProvider::new("shared", 128))
1720 .expect("first registration should succeed");
1721 reg.register(ConstVecProvider::new("other", 64))
1722 .expect("unique registration should succeed");
1723 reg.get_service("shared")
1724 .await
1725 .expect("serving provider should build");
1726 let error = reg
1727 .register(ConstVecProvider::new("shared", 256))
1728 .expect_err("duplicate registration should fail");
1729 assert!(
1730 matches!(error, RuntimeError::InvalidInput(message) if message.contains("shared") && message.contains("serving"))
1731 );
1732
1733 let provider = reg.get_provider("shared").expect("provider must exist");
1734 assert_eq!(provider.dimensions(), 128);
1735 assert_eq!(reg.names(), vec!["shared".to_owned(), "other".to_owned()]);
1736 }
1737
1738 #[tokio::test]
1739 async fn runtime_try_register_embedder_returns_duplicate_error() {
1740 let runtime = crate::KhiveRuntime::memory().expect("memory runtime");
1741 runtime
1742 .try_register_embedder(ConstVecProvider::new("first-custom", 128))
1743 .expect("first registration should succeed");
1744 runtime
1745 .try_register_embedder(ConstVecProvider::new("second-custom", 256))
1746 .expect("second registration should succeed");
1747
1748 let expected = runtime.registered_embedding_model_names();
1749 assert!(expected.ends_with(&["first-custom".to_owned(), "second-custom".to_owned()]));
1750 for _ in 0..100 {
1751 assert_eq!(runtime.registered_embedding_model_names(), expected);
1752 }
1753 runtime
1754 .embedder("first-custom")
1755 .await
1756 .expect("first custom provider should build");
1757
1758 let error = runtime
1759 .try_register_embedder(ConstVecProvider::new("first-custom", 512))
1760 .expect_err("duplicate registration should fail");
1761 assert!(
1762 matches!(error, RuntimeError::InvalidInput(message) if message.contains("first-custom") && message.contains("serving"))
1763 );
1764 }
1765
1766 #[test]
1767 fn bound_engine_list_is_fixed_once_bound() {
1768 let engine = |name: &str, dims: Option<i64>| crate::engine_config::EngineConfig {
1769 name: name.to_owned(),
1770 weight: 1.0,
1771 dims,
1772 };
1773 let mut reg = EmbedderRegistry::new();
1774 for (name, dimensions) in [("bound-a", 8), ("bound-b", 4)] {
1775 reg.register(ConstVecProvider::new(name, dimensions))
1776 .expect("unique registration should succeed");
1777 }
1778 let bound = vec![engine("bound-a", Some(8)), engine("bound-b", None)];
1779 reg.bind_engines(&bound)
1780 .expect("first binding should succeed");
1781 reg.bind_engines(&bound)
1782 .expect("binding the same ordered list again is idempotent");
1783
1784 for different in [
1786 vec![engine("bound-b", None), engine("bound-a", Some(8))],
1787 vec![engine("bound-a", Some(8))],
1788 vec![engine("bound-a", Some(8)), engine("bound-b", Some(4))],
1789 ] {
1790 let error = reg
1791 .bind_engines(&different)
1792 .expect_err("a bound registry must refuse a different list");
1793 assert!(
1794 matches!(&error, RuntimeError::InvalidInput(message) if message.contains("different ordered list")),
1795 "{error:?}"
1796 );
1797 assert_eq!(reg.bound_engines(), Some(bound.as_slice()));
1798 }
1799 }
1800
1801 #[test]
1802 fn names_preserve_registration_order_across_calls() {
1803 let mut reg = EmbedderRegistry::new();
1804 for (name, dimensions) in [("e", 5), ("b", 4), ("d", 3), ("a", 2), ("c", 1)] {
1805 reg.register(ConstVecProvider::new(name, dimensions))
1806 .expect("unique registration should succeed");
1807 }
1808
1809 let expected = ["e", "b", "d", "a", "c"].map(|name| name.to_owned());
1810 for _ in 0..100 {
1811 assert_eq!(reg.names(), expected);
1812 }
1813 }
1814
1815 #[test]
1816 fn names_returns_all_registered() {
1817 let mut reg = EmbedderRegistry::new();
1818 reg.register(ConstVecProvider::new("model-a", 64))
1819 .expect("unique registration should succeed");
1820 reg.register(ConstVecProvider::new("model-b", 128))
1821 .expect("unique registration should succeed");
1822 reg.register(ConstVecProvider::new("model-c", 256))
1823 .expect("unique registration should succeed");
1824
1825 let mut names = reg.names();
1826 names.sort();
1827 assert_eq!(names, vec!["model-a", "model-b", "model-c"]);
1828 }
1829
1830 #[tokio::test]
1831 async fn get_service_unknown_name_returns_error() {
1832 let reg = EmbedderRegistry::new();
1833 let result = reg.get_service("does-not-exist").await;
1834 let err = result.err().expect("expected Err for unknown name, got Ok");
1835 assert!(
1836 matches!(err, RuntimeError::UnknownModel(ref n) if n == "does-not-exist"),
1837 "expected UnknownModel, got {err:?}"
1838 );
1839 }
1840
1841 #[tokio::test]
1842 async fn get_service_calls_build_once() {
1843 let counter = Arc::new(AtomicUsize::new(0));
1844 let provider = ConstVecProvider {
1845 name: "cached-model".to_owned(),
1846 dims: 32,
1847 build_calls: Arc::clone(&counter),
1848 };
1849 let mut reg = EmbedderRegistry::new();
1850 reg.register(provider)
1851 .expect("unique registration should succeed");
1852
1853 let _ = reg.get_service("cached-model").await.unwrap();
1854 let _ = reg.get_service("cached-model").await.unwrap();
1855 let _ = reg.get_service("cached-model").await.unwrap();
1856
1857 assert_eq!(
1858 counter.load(Ordering::SeqCst),
1859 1,
1860 "build must be called exactly once regardless of get_service call count"
1861 );
1862 }
1863
1864 struct SlowBuildProvider {
1865 name: String,
1866 dims: usize,
1867 build_calls: Arc<AtomicUsize>,
1868 }
1869
1870 #[async_trait]
1871 impl EmbedderProvider for SlowBuildProvider {
1872 fn name(&self) -> &str {
1873 &self.name
1874 }
1875
1876 fn dimensions(&self) -> usize {
1877 self.dims
1878 }
1879
1880 async fn build(&self) -> RuntimeResult<Arc<dyn EmbeddingService>> {
1881 self.build_calls.fetch_add(1, Ordering::SeqCst);
1882 tokio::time::sleep(Duration::from_millis(50)).await;
1883 Ok(Arc::new(ConstVecService { dims: self.dims }))
1884 }
1885 }
1886
1887 #[tokio::test(flavor = "multi_thread", worker_threads = 8)]
1888 async fn concurrent_cold_resolutions_single_flight_one_build() {
1889 const CALLERS: usize = 16;
1890 let counter = Arc::new(AtomicUsize::new(0));
1891 let mut reg = EmbedderRegistry::new();
1892 reg.register(SlowBuildProvider {
1893 name: "cold-model".to_owned(),
1894 dims: 8,
1895 build_calls: Arc::clone(&counter),
1896 })
1897 .expect("unique registration should succeed");
1898 let reg = Arc::new(reg);
1899
1900 let mut callers = Vec::with_capacity(CALLERS);
1901 for _ in 0..CALLERS {
1902 let reg = Arc::clone(®);
1903 callers.push(tokio::spawn(
1904 async move { reg.get_service("cold-model").await },
1905 ));
1906 }
1907
1908 for caller in callers {
1909 let service = caller
1910 .await
1911 .expect("resolution task must not panic")
1912 .expect("every concurrent cold resolution must receive a working service");
1913 assert_eq!(service.name(), "const-vec-service");
1914 }
1915
1916 assert_eq!(
1917 counter.load(Ordering::SeqCst),
1918 1,
1919 "concurrent cold resolutions must share a single in-flight build()"
1920 );
1921 }
1922}