khive-runtime 0.17.0

Composable Service API: entity/note CRUD, graph traversal, hybrid search, curation.
Documentation
use super::tests::{poll_once, wait_for_entered, BlockingTestService, ReleaseWorkerOnDrop};
use super::*;
use std::task::Poll;
use std::time::Duration;

type Call = (EmbeddingRole, EmbeddingModel, Vec<String>);

struct CountService {
    blocker: Arc<BlockingTestService>,
    counts: Mutex<Vec<Call>>,
    vectors: Mutex<Vec<Call>>,
}

impl CountService {
    fn new() -> Self {
        Self {
            blocker: Arc::new(BlockingTestService::new()),
            counts: Mutex::new(Vec::new()),
            vectors: Mutex::new(Vec::new()),
        }
    }

    async fn vectors(
        &self,
        texts: &[String],
        model: EmbeddingModel,
        role: EmbeddingRole,
    ) -> lattice_embed::Result<Vec<Vec<f32>>> {
        self.vectors
            .lock()
            .unwrap()
            .push((role, model, texts.to_vec()));
        self.blocker.embed(texts, model).await?;
        Ok(texts.iter().map(|text| vec![text.len() as f32]).collect())
    }
}

#[async_trait]
impl EmbeddingService for CountService {
    async fn embed(
        &self,
        texts: &[String],
        model: EmbeddingModel,
    ) -> lattice_embed::Result<Vec<Vec<f32>>> {
        self.vectors(texts, model, EmbeddingRole::Generic).await
    }

    async fn embed_query(
        &self,
        texts: &[String],
        model: EmbeddingModel,
    ) -> lattice_embed::Result<Vec<Vec<f32>>> {
        self.vectors(texts, model, EmbeddingRole::Query).await
    }

    async fn embed_passage(
        &self,
        texts: &[String],
        model: EmbeddingModel,
    ) -> lattice_embed::Result<Vec<Vec<f32>>> {
        self.vectors(texts, model, EmbeddingRole::Passage).await
    }

    async fn count_tokens(
        &self,
        texts: &[String],
        model: EmbeddingModel,
        role: EmbeddingRole,
    ) -> lattice_embed::Result<Vec<TokenCount>> {
        self.counts
            .lock()
            .unwrap()
            .push((role, model, texts.to_vec()));
        match texts[0].as_str() {
            "unsupported" => {
                return Err(lattice_embed::EmbedError::Unsupported(
                    "count sentinel".into(),
                ))
            }
            "failed" => {
                return Err(lattice_embed::EmbedError::InferenceFailed(
                    "count sentinel".into(),
                ))
            }
            _ => {}
        }
        self.blocker.embed(texts, model).await?;
        Ok(texts
            .iter()
            .map(|text| TokenCount::new(text.len(), text.len().min(3)))
            .collect())
    }

    fn supports_model(&self, _model: EmbeddingModel) -> bool {
        true
    }

    fn name(&self) -> &'static str {
        "token-count-test"
    }
}

#[tokio::test]
async fn token_counts_keep_role_model_and_share_the_native_worker() {
    let inner = Arc::new(CountService::new());
    inner.blocker.release();
    let service = BlockingEmbeddingService::new(Arc::clone(&inner));
    let model = EmbeddingModel::MultilingualE5Small;
    let texts = vec!["ab".to_owned(), "longer".to_owned()];
    let roles = [
        EmbeddingRole::Generic,
        EmbeddingRole::Query,
        EmbeddingRole::Passage,
    ];
    for role in roles {
        assert_eq!(
            service.count_tokens(&texts, model, role).await.unwrap(),
            vec![TokenCount::new(2, 2), TokenCount::new(6, 3)]
        );
    }
    assert_eq!(
        service.embed(&texts, model).await.unwrap(),
        vec![vec![2.0], vec![6.0]]
    );
    assert_eq!(
        *inner.counts.lock().unwrap(),
        roles.map(|role| (role, model, texts.clone()))
    );
    let threads = inner.blocker.thread_ids.lock().unwrap();
    assert_eq!(threads.len(), 1, "counts and vectors share one worker");
    assert!(
        !threads.contains(&std::thread::current().id()),
        "native counts must not run on the caller"
    );
}

#[tokio::test]
async fn cached_reports_recount_tokens_and_reuse_ordered_vectors() {
    let inner = Arc::new(CountService::new());
    inner.blocker.release();
    let service = cached_blocking_service(Arc::clone(&inner));
    let model = EmbeddingModel::MultilingualE5Small;
    let warm = vec!["later".to_owned()];
    for _ in 0..2 {
        let report = service
            .embed_with_report(&warm, model, EmbeddingRole::Passage)
            .await
            .unwrap();
        assert_eq!(report.embeddings, vec![vec![5.0]]);
        assert_eq!(report.token_counts, vec![TokenCount::new(5, 3)]);
    }
    let mixed = vec!["bb".to_owned(), "later".to_owned(), "ccc".to_owned()];
    let report = service
        .embed_with_report(&mixed, model, EmbeddingRole::Passage)
        .await
        .unwrap();
    assert_eq!(report.embeddings, vec![vec![2.0], vec![5.0], vec![3.0]]);
    assert_eq!(
        report.token_counts,
        vec![
            TokenCount::new(2, 2),
            TokenCount::new(5, 3),
            TokenCount::new(3, 3)
        ]
    );
    assert_eq!(
        *inner.counts.lock().unwrap(),
        vec![
            (EmbeddingRole::Passage, model, warm.clone()),
            (EmbeddingRole::Passage, model, warm.clone()),
            (EmbeddingRole::Passage, model, mixed)
        ]
    );
    assert_eq!(
        *inner.vectors.lock().unwrap(),
        vec![
            (EmbeddingRole::Passage, model, warm),
            (
                EmbeddingRole::Passage,
                model,
                vec!["bb".to_owned(), "ccc".to_owned()]
            )
        ]
    );
}

#[tokio::test]
async fn count_worker_preserves_errors_and_rejects_input_and_byte_overflow() {
    let inner = Arc::new(CountService::new());
    inner.blocker.release();
    let service = BlockingEmbeddingService::new(Arc::clone(&inner));
    let model = EmbeddingModel::default();
    let role = EmbeddingRole::Passage;
    assert!(matches!(
        service.count_tokens(&["unsupported".to_owned()], model, role).await,
        Err(lattice_embed::EmbedError::Unsupported(message)) if message == "count sentinel"
    ));
    assert!(matches!(
        service.count_tokens(&["failed".to_owned()], model, role).await,
        Err(lattice_embed::EmbedError::InferenceFailed(message)) if message == "count sentinel"
    ));
    assert!(matches!(
        service.count_tokens(&[], model, role).await,
        Err(lattice_embed::EmbedError::InvalidInput(_))
    ));
    assert!(matches!(
        service
            .count_tokens(&["x".repeat(MAX_TEXT_BYTES + 1)], model, role)
            .await,
        Err(lattice_embed::EmbedError::TextTooLong { .. })
    ));
    assert!(matches!(
        service
            .count_tokens(
                &vec!["x".to_owned(); DEFAULT_MAX_BATCH_SIZE + 1],
                model,
                role
            )
            .await,
        Err(lattice_embed::EmbedError::InvalidInput(_))
    ));
    let bounded = BlockingEmbeddingService::with_byte_budget(Arc::clone(&inner), 3);
    assert!(matches!(
        bounded.count_tokens(&["four".to_owned()], model, role).await,
        Err(lattice_embed::EmbedError::Internal(message)) if message.contains("byte budget exceeded")
    ));
    assert_eq!(
        inner.counts.lock().unwrap().len(),
        2,
        "invalid inputs never reach the provider"
    );
    assert!(inner.vectors.lock().unwrap().is_empty());
    assert_eq!(bounded.in_flight_bytes.load(Ordering::Acquire), 0);
}

#[tokio::test]
async fn token_count_queue_is_bounded_and_discards_cancelled_jobs() {
    let inner = Arc::new(CountService::new());
    let _release = ReleaseWorkerOnDrop(Arc::clone(&inner.blocker));
    let service = BlockingEmbeddingService::new(Arc::clone(&inner));
    let model = EmbeddingModel::default();
    let role = EmbeddingRole::Passage;
    let first_text = vec!["held".to_owned()];
    let mut first = Box::pin(service.count_tokens(&first_text, model, role));
    assert!(poll_once(first.as_mut()).await.is_pending());
    wait_for_entered(&inner.blocker, 1).await;
    let texts: Vec<_> = (0..EMBEDDING_QUEUE_CAPACITY)
        .map(|i| vec![format!("queued-{i}")])
        .collect();
    let mut queued = Vec::new();
    for text in &texts {
        let mut call = Box::pin(service.count_tokens(text, model, role));
        assert!(poll_once(call.as_mut()).await.is_pending());
        queued.push(call);
    }
    assert_eq!(
        service.in_flight_bytes.load(Ordering::Acquire),
        4 + texts.iter().flatten().map(String::len).sum::<usize>()
    );
    assert!(matches!(
        service.count_tokens(&["overflow".to_owned()], model, role).await,
        Err(lattice_embed::EmbedError::Internal(message)) if message == "embedding worker queue is full"
    ));
    drop(queued);
    inner.blocker.release();
    assert_eq!(first.await.unwrap(), vec![TokenCount::new(4, 3)]);
    let later = vec!["later".to_owned()];
    assert_eq!(
        with_embedding_admission(service.count_tokens(&later, model, role))
            .await
            .unwrap(),
        vec![TokenCount::new(5, 3)]
    );
    assert_eq!(
        *inner.counts.lock().unwrap(),
        vec![(role, model, first_text), (role, model, later)]
    );
    tokio::time::timeout(Duration::from_secs(5), async {
        while service.in_flight_bytes.load(Ordering::Acquire) != 0 {
            tokio::task::yield_now().await;
        }
    })
    .await
    .expect("all completed or cancelled count jobs release their bytes");
    assert!(inner.vectors.lock().unwrap().is_empty());
}

#[tokio::test(start_paused = true)]
async fn accepted_count_finishes_but_expired_vector_admission_is_refused() {
    let inner = Arc::new(CountService::new());
    let _release = ReleaseWorkerOnDrop(Arc::clone(&inner.blocker));
    let service = cached_blocking_service(Arc::clone(&inner));
    let texts = vec!["held".to_owned()];
    let model = EmbeddingModel::default();
    let mut call = Box::pin(khive_storage::scope_request_read_deadline(
        Duration::from_millis(100),
        with_embedding_admission(service.embed_with_report(&texts, model, EmbeddingRole::Passage)),
    ));
    assert!(poll_once(call.as_mut()).await.is_pending());
    wait_for_entered(&inner.blocker, 1).await;
    tokio::time::advance(Duration::from_millis(100)).await;
    assert!(
        poll_once(call.as_mut()).await.is_pending(),
        "an accepted count is not cancelled at expiry"
    );
    inner.blocker.release();
    let watchdog = std::time::Instant::now() + Duration::from_secs(5);
    let result = loop {
        if let Poll::Ready(result) = poll_once(call.as_mut()).await {
            break result;
        }
        assert!(
            std::time::Instant::now() < watchdog,
            "second admission must not hang after the count finishes"
        );
        tokio::task::yield_now().await;
    };
    assert!(matches!(
        result,
        Err(RuntimeError::Storage(
            khive_storage::StorageError::AdmissionTimeout { .. }
        ))
    ));
    assert_eq!(inner.counts.lock().unwrap().len(), 1);
    assert!(
        inner.vectors.lock().unwrap().is_empty(),
        "expired second admission never starts inference"
    );
}

#[tokio::test]
async fn warm_report_still_needs_count_admission_but_warm_vectors_do_not() {
    let inner = Arc::new(CountService::new());
    inner.blocker.release();
    let service = cached_blocking_service(Arc::clone(&inner));
    let texts = vec!["later".to_owned()];
    let model = EmbeddingModel::default();
    service
        .embed_with_report(&texts, model, EmbeddingRole::Passage)
        .await
        .unwrap();
    let report = khive_storage::scope_request_read_deadline(
        Duration::ZERO,
        with_embedding_admission(service.embed_with_report(&texts, model, EmbeddingRole::Passage)),
    )
    .await;
    assert!(matches!(
        report,
        Err(RuntimeError::Storage(
            khive_storage::StorageError::AdmissionTimeout { .. }
        ))
    ));
    let vectors = khive_storage::scope_request_read_deadline(
        Duration::ZERO,
        with_embedding_admission(service.embed_passage(&texts, model)),
    )
    .await
    .unwrap();
    assert_eq!(vectors, vec![vec![5.0]]);
    assert_eq!(
        inner.counts.lock().unwrap().len(),
        1,
        "expired count never enters worker"
    );
    assert_eq!(
        inner.vectors.lock().unwrap().len(),
        1,
        "cached vector adds no native work"
    );
}