Skip to main content

everruns_turbopuffer/
lib.rs

1#![cfg_attr(test, allow(clippy::unwrap_used, clippy::expect_used))]
2//! Turbopuffer vector-store backend for Knowledge Indexes.
3//!
4//! Implements [`everruns_contracts::vector_store::VectorStore`] against Turbopuffer's
5//! v2 HTTP API (<https://turbopuffer.com/docs>). This is the reference
6//! production backend; the in-memory store stays the default and Turbopuffer is
7//! opt-in via `TURBOPUFFER_API_KEY`.
8//!
9//! Part of the [Everruns](https://everruns.com) ecosystem.
10//!
11//! ```rust
12//! use everruns_turbopuffer::TurbopufferVectorStore;
13//! let _store = TurbopufferVectorStore::new("https://example.turbopuffer.com", "api-key");
14//! ```
15//!
16//! Multitenancy and multi-index isolation are handled by the caller: each
17//! Knowledge Index gets one org-prefixed namespace (see
18//! [`everruns_contracts::vector_store::index_namespace`]), and this backend simply
19//! maps each namespace string onto a Turbopuffer namespace path segment.
20//!
21//! ## Error handling
22//!
23//! Non-2xx responses are mapped to sanitized [`anyhow`] errors. The API key is
24//! only ever sent in the `Authorization` header and is **never** included in any
25//! error message or log line.
26
27use anyhow::{Context, Result, bail};
28use async_trait::async_trait;
29use everruns_contracts::vector_store::{VectorMatch, VectorQuery, VectorRecord, VectorStore};
30use serde::Deserialize;
31use serde_json::{Value, json};
32use sha2::{Digest, Sha256};
33use std::sync::Arc;
34
35/// Provider-reported resources, before monetary pricing. Missing receipts must
36/// never be replaced with guessed request sizes or a fabricated query count.
37#[derive(Debug, Clone, Default, Deserialize)]
38pub struct Usage {
39    #[serde(default, rename = "billable_logical_bytes_written")]
40    pub written_bytes: u64,
41    #[serde(default, rename = "billable_logical_bytes_queried")]
42    pub queried_bytes: u64,
43    #[serde(default, rename = "billable_logical_bytes_returned")]
44    pub returned_bytes: u64,
45}
46
47#[async_trait]
48pub trait UsageRecorder: Send + Sync {
49    async fn record(&self, source_ref: &str, usage: Usage) -> Result<()>;
50}
51
52/// Distance metric used for all namespaces. Cosine matches the in-memory
53/// backend's ranking so the two stores rank embeddings the same way.
54const DISTANCE_METRIC: &str = "cosine_distance";
55
56/// Turbopuffer-backed [`VectorStore`].
57///
58/// Cheap to clone is unnecessary; construct once and share behind an `Arc` (as
59/// the platform definition does).
60pub struct TurbopufferVectorStore {
61    /// Regional base URL, e.g. `https://gcp-us-central1.turbopuffer.com`. No
62    /// trailing slash.
63    base_url: String,
64    api_key: String,
65    usage_recorder: Option<Arc<dyn UsageRecorder>>,
66}
67
68impl TurbopufferVectorStore {
69    /// Build a new backend. `base_url` is the regional Turbopuffer endpoint
70    /// (any trailing slash is trimmed); `api_key` authenticates every request.
71    pub fn new(base_url: impl Into<String>, api_key: impl Into<String>) -> Self {
72        // EVE-924: choose the rustls backend on the startup path. The shared
73        // client installs it as well, but that now happens on the first
74        // request, and products expect the process-wide choice to be settled
75        // while providers are being constructed.
76        everruns_contracts::install_default_crypto_provider();
77        Self {
78            base_url: base_url.into().trim_end_matches('/').to_string(),
79            api_key: api_key.into(),
80            usage_recorder: None,
81        }
82    }
83
84    pub fn with_usage_recorder(mut self, recorder: Arc<dyn UsageRecorder>) -> Self {
85        self.usage_recorder = Some(recorder);
86        self
87    }
88
89    /// The process-wide request HTTP client, resolved per request rather than
90    /// held as a field. Building it loads the platform trust store (~1.3 ms),
91    /// which would otherwise land in `new`; after the first request this is a
92    /// `OnceLock` read and an `Arc` clone.
93    ///
94    /// EVE-635: the shared client carries connect + overall request timeouts so
95    /// a hung vector-store read cannot block indefinitely.
96    fn http(&self) -> reqwest::Client {
97        everruns_contracts::driver_helpers::shared_request_http_client()
98    }
99
100    fn namespace_url(&self, namespace: &str) -> String {
101        format!("{}/v2/namespaces/{namespace}", self.base_url)
102    }
103
104    /// POST a JSON body to the namespace write endpoint and require a 2xx.
105    async fn post_write(&self, namespace: &str, body: Value, op: &str) -> Result<()> {
106        let resp = self
107            .http()
108            .post(self.namespace_url(namespace))
109            .bearer_auth(&self.api_key)
110            .json(&body)
111            .send()
112            .await
113            .with_context(|| format!("turbopuffer {op} request failed"))?;
114        let resp = ensure_success(resp, op).await?;
115        if op == "upsert"
116            && let Some(recorder) = &self.usage_recorder
117        {
118            let response: Value = resp
119                .json()
120                .await
121                .context("turbopuffer write receipt invalid")?;
122            let usage = response
123                .get("billing")
124                .cloned()
125                .ok_or_else(|| anyhow::anyhow!("turbopuffer write receipt missing"))?;
126            // Stable across worker retries of the same immutable fragment batch.
127            // Cleanup/reconciliation writes are platform overhead, not charges.
128            let mut digest = Sha256::new();
129            digest.update(namespace.as_bytes());
130            digest.update(serde_json::to_vec(&body)?);
131            let source_ref = format!("knowledge:index-write:{}", hex::encode(digest.finalize()));
132            recorder
133                .record(
134                    &source_ref,
135                    parse_receipt(usage, &["billable_logical_bytes_written"])?,
136                )
137                .await?;
138        }
139        Ok(())
140    }
141}
142
143#[async_trait]
144impl VectorStore for TurbopufferVectorStore {
145    async fn upsert(&self, namespace: &str, records: Vec<VectorRecord>) -> Result<()> {
146        if records.is_empty() {
147            return Ok(());
148        }
149        // Derive the concrete vector dimension from the records so the schema
150        // declares `[N]f32`. All records in one index share `vector_dim`.
151        let dim = records[0].vector.len();
152        let rows: Vec<Value> = records
153            .into_iter()
154            .map(|r| {
155                json!({
156                    "id": r.id,
157                    "vector": r.vector,
158                    "text": r.text,
159                    "document_id": r.document_id,
160                })
161            })
162            .collect();
163
164        // Send the schema on every upsert so ANN + BM25 are enabled. Turbopuffer
165        // treats the schema as idempotent; re-sending it is a no-op once set.
166        let body = json!({
167            "upsert_rows": rows,
168            "distance_metric": DISTANCE_METRIC,
169            "schema": {
170                "text": { "type": "string", "full_text_search": true },
171                "vector": { "type": format!("[{dim}]f32"), "ann": true },
172                "document_id": { "type": "string" },
173            },
174        });
175        let mut body = body;
176        if dim == 0 {
177            body.as_object_mut().map(|v| v.remove("distance_metric"));
178            body["schema"].as_object_mut().map(|v| v.remove("vector"));
179            if let Some(rows) = body["upsert_rows"].as_array_mut() {
180                for row in rows {
181                    row.as_object_mut().map(|v| v.remove("vector"));
182                }
183            }
184        }
185        self.post_write(namespace, body, "upsert").await
186    }
187
188    async fn query(&self, namespace: &str, query: VectorQuery) -> Result<Vec<VectorMatch>> {
189        if query.top_k == 0 {
190            return Ok(Vec::new());
191        }
192        let top_k = query.top_k;
193        let vector = query.vector.filter(|v| !v.is_empty());
194        let text = query
195            .text
196            .map(|t| t.trim().to_string())
197            .filter(|t| !t.is_empty());
198
199        // Pick a query shape based on the available signals and remember whether
200        // the response `$dist` is a raw ANN distance (lower = better, must be
201        // negated) or a relevance score (BM25 / RRF; higher = better).
202        let (body, dist_is_distance) = match (vector, text) {
203            // Hybrid: multi-query (ANN + BM25) fused server-side with RRF. The
204            // fused `$dist` is an RRF score where higher is more relevant.
205            (Some(v), Some(t)) => (
206                json!({
207                    "queries": [
208                        { "rank_by": ["vector", "ANN", v], "top_k": top_k,
209                          "include_attributes": ["document_id"] },
210                        { "rank_by": ["text", "BM25", t], "top_k": top_k,
211                          "include_attributes": ["document_id"] },
212                    ],
213                    "rerank_by": ["RRF"],
214                }),
215                false,
216            ),
217            // Vector only: raw ANN with cosine_distance. `$dist` is a distance
218            // where lower = closer, so we negate it into `score`.
219            (Some(v), None) => (
220                json!({
221                    "rank_by": ["vector", "ANN", v],
222                    "top_k": top_k,
223                    "include_attributes": ["document_id"],
224                }),
225                true,
226            ),
227            // Text only: BM25 relevance score, higher = better.
228            (None, Some(t)) => (
229                json!({
230                    "rank_by": ["text", "BM25", t],
231                    "top_k": top_k,
232                    "include_attributes": ["document_id"],
233                }),
234                false,
235            ),
236            // No signal: nothing to rank against.
237            (None, None) => return Ok(Vec::new()),
238        };
239
240        let resp = self
241            .http()
242            .post(format!("{}/query", self.namespace_url(namespace)))
243            .bearer_auth(&self.api_key)
244            .json(&body)
245            .send()
246            .await
247            .context("turbopuffer query request failed")?;
248        let resp = ensure_success(resp, "query").await?;
249        let parsed: QueryResponse = resp
250            .json()
251            .await
252            .context("turbopuffer query: malformed response body")?;
253        if let Some(recorder) = &self.usage_recorder {
254            let usage = parsed
255                .billing
256                .clone()
257                .ok_or_else(|| anyhow::anyhow!("turbopuffer query receipt missing"))?;
258            // Separate actual queries are separate supplier operations, even
259            // when their text is identical. Replay of a receipt never bills twice.
260            let source_ref = format!("knowledge:index-query:{}", uuid::Uuid::now_v7());
261            recorder
262                .record(
263                    &source_ref,
264                    parse_receipt(
265                        usage,
266                        &[
267                            "billable_logical_bytes_queried",
268                            "billable_logical_bytes_returned",
269                        ],
270                    )?,
271                )
272                .await?;
273        }
274
275        // Both single- and multi-query (RRF) responses surface a fused `rows`
276        // array already ordered best-first. Map `$dist` to `VectorMatch.score`
277        // so ordering is best-first regardless of mode: negate for raw ANN
278        // distance (lower distance => higher score), pass through for BM25/RRF
279        // relevance scores. We preserve the server's ordering and only take the
280        // requested `top_k`.
281        let matches = parsed
282            .rows
283            .into_iter()
284            .take(top_k)
285            .map(|row| {
286                let raw = row.dist.unwrap_or(0.0);
287                let score = if dist_is_distance { -raw } else { raw };
288                VectorMatch {
289                    id: row.id,
290                    document_id: row.document_id.unwrap_or_default(),
291                    score,
292                }
293            })
294            .collect();
295        Ok(matches)
296    }
297
298    async fn delete_by_document(&self, namespace: &str, document_id: &str) -> Result<()> {
299        let body = json!({
300            "delete_by_filter": ["document_id", "Eq", document_id],
301        });
302        self.post_write(namespace, body, "delete_by_document").await
303    }
304
305    async fn delete_namespace(&self, namespace: &str) -> Result<()> {
306        let resp = self
307            .http()
308            .delete(self.namespace_url(namespace))
309            .bearer_auth(&self.api_key)
310            .send()
311            .await
312            .context("turbopuffer delete_namespace request failed")?;
313        // Idempotent: deleting an already-absent namespace is success.
314        if resp.status() == reqwest::StatusCode::NOT_FOUND {
315            return Ok(());
316        }
317        ensure_success(resp, "delete_namespace").await?;
318        Ok(())
319    }
320}
321
322/// A single result row from the query endpoint. Turbopuffer keys the distance
323/// as `$dist`; requested attributes (here `document_id`) are flattened onto the
324/// row alongside `id`.
325#[derive(Debug, Deserialize)]
326struct QueryRow {
327    id: String,
328    #[serde(rename = "$dist")]
329    dist: Option<f32>,
330    #[serde(default)]
331    document_id: Option<String>,
332}
333
334#[derive(Debug, Deserialize)]
335struct QueryResponse {
336    #[serde(default)]
337    billing: Option<Value>,
338    #[serde(default)]
339    rows: Vec<QueryRow>,
340}
341
342fn parse_receipt(billing: Value, required: &[&str]) -> Result<Usage> {
343    // Providers report different fields for writes and queries. Require the
344    // operation's fields, including explicit zeroes; absent data is not free use.
345    for field in required {
346        billing
347            .get(field)
348            .and_then(Value::as_u64)
349            .context("turbopuffer billing receipt incomplete")?;
350    }
351    serde_json::from_value(billing).context("turbopuffer billing receipt invalid")
352}
353
354/// Require a 2xx status, returning a sanitized error otherwise.
355///
356/// The API key lives only in the request `Authorization` header and is never
357/// part of the response body or the error we build here, so error text is safe
358/// to surface and log.
359async fn ensure_success(resp: reqwest::Response, op: &str) -> Result<reqwest::Response> {
360    let status = resp.status();
361    if status.is_success() {
362        return Ok(resp);
363    }
364    // Read the body for diagnostics; Turbopuffer returns a JSON error message.
365    // It never echoes the API key.
366    let body = resp.text().await.unwrap_or_default();
367    let detail = sanitize_error_body(&body);
368    bail!("turbopuffer {op} failed: HTTP {status}{detail}");
369}
370
371/// Trim and bound the error body so we never log unbounded server output.
372fn sanitize_error_body(body: &str) -> String {
373    let trimmed = body.trim();
374    if trimmed.is_empty() {
375        return String::new();
376    }
377    const MAX: usize = 500;
378    let snippet: String = trimmed.chars().take(MAX).collect();
379    format!(": {snippet}")
380}
381
382#[cfg(test)]
383mod tests {
384    use super::*;
385    use wiremock::matchers::{body_json, method, path};
386    use wiremock::{Mock, MockServer, Request, ResponseTemplate};
387
388    #[derive(Default)]
389    struct Receipts(std::sync::Mutex<Vec<(String, Usage)>>);
390
391    #[async_trait]
392    impl UsageRecorder for Receipts {
393        async fn record(&self, source_ref: &str, usage: Usage) -> Result<()> {
394            self.0.lock().unwrap().push((source_ref.to_string(), usage));
395            Ok(())
396        }
397    }
398
399    #[tokio::test]
400    async fn managed_operations_reject_incomplete_receipts_without_recording_zero_usage() {
401        let server = MockServer::start().await;
402        let receipts = Arc::new(Receipts::default());
403        let store = TurbopufferVectorStore::new(server.uri(), "fixture-key")
404            .with_usage_recorder(receipts.clone());
405        Mock::given(method("POST"))
406            .respond_with(ResponseTemplate::new(200).set_body_json(json!({
407                "rows":[], "billing":{}
408            })))
409            .mount(&server)
410            .await;
411        assert!(
412            store
413                .upsert(
414                    "receipt-test",
415                    vec![record("fragment", "version", vec![], "passage")]
416                )
417                .await
418                .is_err()
419        );
420        assert!(
421            store
422                .query(
423                    "receipt-test",
424                    VectorQuery {
425                        vector: None,
426                        text: Some("passage".into()),
427                        top_k: 1
428                    }
429                )
430                .await
431                .is_err()
432        );
433        assert!(receipts.0.lock().unwrap().is_empty());
434    }
435
436    #[tokio::test]
437    async fn write_receipts_preserve_provider_units_and_replay_identity() {
438        let server = MockServer::start().await;
439        let receipts = Arc::new(Receipts::default());
440        let store = TurbopufferVectorStore::new(server.uri(), "fixture-key")
441            .with_usage_recorder(receipts.clone());
442        Mock::given(method("POST"))
443            .and(path("/v2/namespaces/receipt-test"))
444            .respond_with(ResponseTemplate::new(200).set_body_json(json!({
445                "billing":{"billable_logical_bytes_written":23456}
446            })))
447            .mount(&server)
448            .await;
449        let records = vec![record("fragment", "version", vec![1.0, 0.0], "passage")];
450        store.upsert("receipt-test", records.clone()).await.unwrap();
451        store.upsert("receipt-test", records).await.unwrap();
452        let receipts = receipts.0.lock().unwrap();
453        assert_eq!(receipts.len(), 2);
454        assert_eq!(
455            receipts[0].0, receipts[1].0,
456            "an immutable batch retry has one billing identity"
457        );
458        assert_eq!(
459            receipts[0].1.written_bytes, 23456,
460            "supplier units, not JSON payload size"
461        );
462    }
463
464    #[tokio::test]
465    async fn query_receipts_count_empty_results_and_keep_separate_actual_queries_distinct() {
466        let server = MockServer::start().await;
467        let receipts = Arc::new(Receipts::default());
468        let store = TurbopufferVectorStore::new(server.uri(), "fixture-key")
469            .with_usage_recorder(receipts.clone());
470        Mock::given(method("POST")).and(path("/v2/namespaces/receipt-test/query"))
471            .respond_with(ResponseTemplate::new(200).set_body_json(json!({
472                "rows":[],"billing":{"billable_logical_bytes_queried":1280000000u64,"billable_logical_bytes_returned":0}
473            }))).mount(&server).await;
474        for _ in 0..2 {
475            assert!(
476                store
477                    .query(
478                        "receipt-test",
479                        VectorQuery {
480                            vector: None,
481                            text: Some("same query".into()),
482                            top_k: 5
483                        }
484                    )
485                    .await
486                    .unwrap()
487                    .is_empty()
488            );
489        }
490        let receipts = receipts.0.lock().unwrap();
491        assert_eq!(receipts.len(), 2);
492        assert_ne!(receipts[0].0, receipts[1].0);
493        assert_eq!(receipts[0].1.queried_bytes, 1280000000);
494    }
495
496    fn record(id: &str, doc: &str, vector: Vec<f32>, text: &str) -> VectorRecord {
497        VectorRecord {
498            id: id.to_string(),
499            vector,
500            text: text.to_string(),
501            document_id: doc.to_string(),
502        }
503    }
504
505    #[tokio::test]
506    async fn upsert_sends_rows_schema_and_distance_metric() {
507        let server = MockServer::start().await;
508        let store = TurbopufferVectorStore::new(server.uri(), "secret-key");
509
510        Mock::given(method("POST"))
511            .and(path("/v2/namespaces/org_1__kidx_a"))
512            .and(body_json(json!({
513                "upsert_rows": [
514                    { "id": "kchk_a", "vector": [1.0, 0.0], "text": "alpha", "document_id": "kidoc_1" },
515                ],
516                "distance_metric": "cosine_distance",
517                "schema": {
518                    "text": { "type": "string", "full_text_search": true },
519                    "vector": { "type": "[2]f32", "ann": true },
520                    "document_id": { "type": "string" },
521                },
522            })))
523            .respond_with(ResponseTemplate::new(200).set_body_json(json!({})))
524            .expect(1)
525            .mount(&server)
526            .await;
527
528        store
529            .upsert(
530                "org_1__kidx_a",
531                vec![record("kchk_a", "kidoc_1", vec![1.0, 0.0], "alpha")],
532            )
533            .await
534            .unwrap();
535    }
536
537    #[tokio::test]
538    async fn lexical_upsert_has_no_vector_schema_or_distance_metric() {
539        let server = MockServer::start().await;
540        let store = TurbopufferVectorStore::new(server.uri(), "secret-key");
541        Mock::given(method("POST")).and(path("/v2/namespaces/knowledge_test"))
542            .and(body_json(json!({"upsert_rows":[{"id":"p1","text":"refund policy","document_id":"v1"}],"schema":{"text":{"type":"string","full_text_search":true},"document_id":{"type":"string"}}})))
543            .respond_with(ResponseTemplate::new(200).set_body_json(json!({}))).expect(1).mount(&server).await;
544        store
545            .upsert(
546                "knowledge_test",
547                vec![record("p1", "v1", vec![], "refund policy")],
548            )
549            .await
550            .unwrap();
551    }
552
553    #[tokio::test]
554    async fn upsert_empty_is_noop() {
555        // No mock mounted: an HTTP call would fail. Empty upsert must not call.
556        let store = TurbopufferVectorStore::new("http://127.0.0.1:1", "secret-key");
557        store.upsert("org_1__kidx_a", vec![]).await.unwrap();
558    }
559
560    #[tokio::test]
561    async fn query_vector_only_negates_distance_into_score() {
562        let server = MockServer::start().await;
563        let store = TurbopufferVectorStore::new(server.uri(), "secret-key");
564
565        Mock::given(method("POST"))
566            .and(path("/v2/namespaces/org_1__kidx_a/query"))
567            .and(body_json(json!({
568                "rank_by": ["vector", "ANN", [1.0, 0.0]],
569                "top_k": 2,
570                "include_attributes": ["document_id"],
571            })))
572            .respond_with(ResponseTemplate::new(200).set_body_json(json!({
573                "rows": [
574                    { "id": "kchk_a", "$dist": 0.1, "document_id": "kidoc_1" },
575                    { "id": "kchk_c", "$dist": 0.4, "document_id": "kidoc_2" },
576                ],
577            })))
578            .expect(1)
579            .mount(&server)
580            .await;
581
582        let matches = store
583            .query(
584                "org_1__kidx_a",
585                VectorQuery {
586                    vector: Some(vec![1.0, 0.0]),
587                    text: None,
588                    top_k: 2,
589                },
590            )
591            .await
592            .unwrap();
593
594        let ids: Vec<_> = matches.iter().map(|m| m.id.as_str()).collect();
595        assert_eq!(ids, vec!["kchk_a", "kchk_c"]);
596        // ANN cosine_distance: lower $dist => higher score (negated), best-first.
597        assert!(matches[0].score > matches[1].score);
598        assert_eq!(matches[0].score, -0.1);
599        assert_eq!(matches[0].document_id, "kidoc_1");
600    }
601
602    #[tokio::test]
603    async fn query_text_only_passes_bm25_score_through() {
604        let server = MockServer::start().await;
605        let store = TurbopufferVectorStore::new(server.uri(), "secret-key");
606
607        Mock::given(method("POST"))
608            .and(path("/v2/namespaces/org_1__kidx_a/query"))
609            .and(body_json(json!({
610                "rank_by": ["text", "BM25", "quick fox"],
611                "top_k": 5,
612                "include_attributes": ["document_id"],
613            })))
614            .respond_with(ResponseTemplate::new(200).set_body_json(json!({
615                "rows": [ { "id": "kchk_a", "$dist": 3.2, "document_id": "kidoc_1" } ],
616            })))
617            .expect(1)
618            .mount(&server)
619            .await;
620
621        let matches = store
622            .query(
623                "org_1__kidx_a",
624                VectorQuery {
625                    vector: None,
626                    text: Some("  quick fox  ".to_string()),
627                    top_k: 5,
628                },
629            )
630            .await
631            .unwrap();
632        assert_eq!(matches.len(), 1);
633        assert_eq!(matches[0].id, "kchk_a");
634        // BM25: higher $dist => higher score (passed through).
635        assert_eq!(matches[0].score, 3.2);
636    }
637
638    #[tokio::test]
639    async fn query_hybrid_uses_multi_query_with_rrf() {
640        let server = MockServer::start().await;
641        let store = TurbopufferVectorStore::new(server.uri(), "secret-key");
642
643        Mock::given(method("POST"))
644            .and(path("/v2/namespaces/org_1__kidx_a/query"))
645            .and(body_json(json!({
646                "queries": [
647                    { "rank_by": ["vector", "ANN", [1.0, 0.0]], "top_k": 2,
648                      "include_attributes": ["document_id"] },
649                    { "rank_by": ["text", "BM25", "database"], "top_k": 2,
650                      "include_attributes": ["document_id"] },
651                ],
652                "rerank_by": ["RRF"],
653            })))
654            .respond_with(ResponseTemplate::new(200).set_body_json(json!({
655                "rows": [
656                    { "id": "kchk_a", "$dist": 0.032, "document_id": "kidoc_1" },
657                    { "id": "kchk_b", "$dist": 0.016, "document_id": "kidoc_1" },
658                ],
659            })))
660            .expect(1)
661            .mount(&server)
662            .await;
663
664        let matches = store
665            .query(
666                "org_1__kidx_a",
667                VectorQuery {
668                    vector: Some(vec![1.0, 0.0]),
669                    text: Some("database".to_string()),
670                    top_k: 2,
671                },
672            )
673            .await
674            .unwrap();
675        // RRF fused $dist: higher => better, preserve server order best-first.
676        assert_eq!(matches[0].id, "kchk_a");
677        assert!(matches[0].score >= matches[1].score);
678    }
679
680    #[tokio::test]
681    async fn delete_by_document_posts_filter() {
682        let server = MockServer::start().await;
683        let store = TurbopufferVectorStore::new(server.uri(), "secret-key");
684
685        Mock::given(method("POST"))
686            .and(path("/v2/namespaces/org_1__kidx_a"))
687            .and(body_json(json!({
688                "delete_by_filter": ["document_id", "Eq", "kidoc_1"],
689            })))
690            .respond_with(ResponseTemplate::new(200).set_body_json(json!({})))
691            .expect(1)
692            .mount(&server)
693            .await;
694
695        store
696            .delete_by_document("org_1__kidx_a", "kidoc_1")
697            .await
698            .unwrap();
699    }
700
701    #[tokio::test]
702    async fn delete_namespace_treats_404_as_success() {
703        let server = MockServer::start().await;
704        let store = TurbopufferVectorStore::new(server.uri(), "secret-key");
705
706        Mock::given(method("DELETE"))
707            .and(path("/v2/namespaces/org_1__missing"))
708            .respond_with(ResponseTemplate::new(404).set_body_json(json!({
709                "error": "namespace not found",
710            })))
711            .expect(1)
712            .mount(&server)
713            .await;
714
715        store.delete_namespace("org_1__missing").await.unwrap();
716    }
717
718    #[tokio::test]
719    async fn non_2xx_fails_closed_without_leaking_api_key() {
720        let server = MockServer::start().await;
721        let store = TurbopufferVectorStore::new(server.uri(), "super-secret-api-key");
722
723        Mock::given(method("POST"))
724            .and(path("/v2/namespaces/org_1__kidx_a"))
725            .respond_with(ResponseTemplate::new(401).set_body_json(json!({
726                "error": "unauthorized",
727            })))
728            .mount(&server)
729            .await;
730
731        let err = store
732            .upsert(
733                "org_1__kidx_a",
734                vec![record("kchk_a", "kidoc_1", vec![1.0, 0.0], "alpha")],
735            )
736            .await
737            .unwrap_err();
738        let msg = format!("{err:#}");
739        assert!(msg.contains("401"), "error should report status: {msg}");
740        assert!(
741            !msg.contains("super-secret-api-key"),
742            "error must not leak the API key: {msg}"
743        );
744    }
745
746    #[tokio::test]
747    async fn query_top_k_zero_skips_request() {
748        // No mock: a request would fail. top_k == 0 must short-circuit.
749        let store = TurbopufferVectorStore::new("http://127.0.0.1:1", "secret-key");
750        let matches = store
751            .query(
752                "org_1__kidx_a",
753                VectorQuery {
754                    vector: Some(vec![1.0]),
755                    text: None,
756                    top_k: 0,
757                },
758            )
759            .await
760            .unwrap();
761        assert!(matches.is_empty());
762    }
763
764    #[tokio::test]
765    async fn auth_header_carries_bearer_key() {
766        let server = MockServer::start().await;
767        let store = TurbopufferVectorStore::new(server.uri(), "secret-key");
768
769        Mock::given(method("DELETE"))
770            .and(path("/v2/namespaces/org_1__kidx_a"))
771            .and(|req: &Request| {
772                req.headers
773                    .get("authorization")
774                    .map(|v| v == "Bearer secret-key")
775                    .unwrap_or(false)
776            })
777            .respond_with(ResponseTemplate::new(200))
778            .expect(1)
779            .mount(&server)
780            .await;
781
782        store.delete_namespace("org_1__kidx_a").await.unwrap();
783    }
784
785    /// Live test: hits the real Turbopuffer API. Gated behind the
786    /// `turbopuffer-live-tests` feature so it never runs in the default suite.
787    /// When the feature is on but `TURBOPUFFER_API_KEY` is missing/empty the
788    /// test fails closed (panics).
789    #[cfg(feature = "turbopuffer-live-tests")]
790    #[tokio::test]
791    async fn live_roundtrip() {
792        let api_key = std::env::var("TURBOPUFFER_API_KEY")
793            .ok()
794            .filter(|k| !k.trim().is_empty())
795            .unwrap_or_else(|| {
796                panic!(
797                    "turbopuffer-live-tests enabled but TURBOPUFFER_API_KEY is missing/empty; \
798                     live tests must fail closed"
799                )
800            });
801        let base_url = std::env::var("TURBOPUFFER_BASE_URL")
802            .unwrap_or_else(|_| "https://gcp-us-central1.turbopuffer.com".to_string());
803        let store = TurbopufferVectorStore::new(base_url, api_key);
804        let ns = "org_test__kidx_live_roundtrip";
805        store.delete_namespace(ns).await.unwrap();
806        store
807            .upsert(
808                ns,
809                vec![record(
810                    "kchk_live",
811                    "kidoc_live",
812                    vec![1.0, 0.0],
813                    "alpha live",
814                )],
815            )
816            .await
817            .unwrap();
818        store.delete_namespace(ns).await.unwrap();
819    }
820}