1use std::collections::HashMap;
4use std::path::PathBuf;
5
6use serde_json::{json, Value};
7
8use crate::client::Client;
9use crate::errors::{Error, Result};
10use crate::ingestion::IngestFilesOptions;
11use crate::intelligence::AskOptions;
12use crate::secrets::OPENAI_API_KEY_SECRET_REF;
13use crate::sources::IntoCreateSourceRequest;
14use crate::transport::Request;
15use crate::types::{
16 infer_embedding_dim, AddTextsResponse, AskResponse, CreateDatasetRequest, DatasetDocumentList,
17 DatasetInfo, DatasetList, DeleteVectorsRequest, DeleteVectorsResponse, DocumentListOpts,
18 EmbedRequest, EmbedResponse, EmbeddingConfig, InsertVectorsRequest, InsertVectorsResponse, Job,
19 Metadata, MetadataSchemaField, MetadataSchemaUpdateMode, Rerank, RerankConfig, SearchRequest,
20 SearchResponse, TextDocument, Vector, VectorId, DEFAULT_EMBEDDING_MODEL,
21 DEFAULT_EMBEDDING_PROVIDER,
22};
23
24pub const DEFAULT_SEARCH_TOP_K: u32 = 10;
26
27pub const DEFAULT_PAGE_LIMIT: u32 = 50;
29
30#[derive(Debug, Clone, Copy, Default)]
37pub struct Pagination {
38 pub limit: Option<u32>,
39 pub offset: Option<u32>,
40}
41
42impl Pagination {
43 pub(crate) fn resolve(self) -> (u32, u32) {
46 (
47 self.limit.unwrap_or(DEFAULT_PAGE_LIMIT),
48 self.offset.unwrap_or(0),
49 )
50 }
51}
52
53impl From<()> for Pagination {
54 fn from(_: ()) -> Self {
55 Pagination::default()
56 }
57}
58
59impl From<u32> for Pagination {
60 fn from(limit: u32) -> Self {
61 Pagination {
62 limit: Some(limit),
63 offset: None,
64 }
65 }
66}
67
68impl From<(u32, u32)> for Pagination {
69 fn from((limit, offset): (u32, u32)) -> Self {
70 Pagination {
71 limit: Some(limit),
72 offset: Some(offset),
73 }
74 }
75}
76
77#[derive(Clone)]
79pub struct DatasetService {
80 client: Client,
81}
82
83impl DatasetService {
84 pub(crate) fn new(client: Client) -> Self {
85 Self { client }
86 }
87
88 pub async fn list<P: Into<Pagination>>(&self, pagination: P) -> Result<DatasetList> {
103 let (limit, offset) = pagination.into().resolve();
104 let mut req = Request {
105 method: "GET".into(),
106 path: "/datasets".into(),
107 ..Default::default()
108 };
109 push_pagination(&mut req.query, limit, offset);
110 self.client.dispatcher().json(req).await
111 }
112
113 pub async fn get(&self, dataset_id: &str) -> Result<Dataset> {
115 let info: DatasetInfo = self
116 .client
117 .dispatcher()
118 .json(Request {
119 method: "GET".into(),
120 path: format!("/datasets/{dataset_id}"),
121 ..Default::default()
122 })
123 .await?;
124 Ok(Dataset::new(self.client.clone(), info))
125 }
126
127 pub async fn create<R: Into<CreateDatasetRequest>>(&self, request: R) -> Result<Dataset> {
153 let mut request = request.into();
154 self.apply_create_defaults(&mut request)?;
155
156 let mut body = serde_json::to_value(&request)?;
157 if let Value::Object(ref mut map) = body {
158 map.insert("index_type".into(), Value::String("sable".into()));
159 }
160 let info: DatasetInfo = self
161 .client
162 .dispatcher()
163 .json(Request {
164 method: "POST".into(),
165 path: "/datasets".into(),
166 body: Some(body),
167 ..Default::default()
168 })
169 .await?;
170 Ok(Dataset::new(self.client.clone(), info))
171 }
172
173 pub async fn create_with_openai_api_key<R: Into<CreateDatasetRequest>, S: Into<String>>(
176 &self,
177 request: R,
178 api_key: S,
179 ) -> Result<Dataset> {
180 self.client
181 .org_secrets()
182 .put_openai_api_key(api_key.into())
183 .await?;
184 let mut request = request.into();
185 let embedding = request
186 .embedding
187 .get_or_insert_with(|| EmbeddingConfig::openai("small"));
188 embedding.provider = Some("openai".into());
189 if embedding.model.is_none() {
190 embedding.model = Some("text-embedding-3-small".into());
191 }
192 embedding.secret_ref = Some(OPENAI_API_KEY_SECRET_REF.into());
193 self.create(request).await
194 }
195
196 fn apply_create_defaults(&self, request: &mut CreateDatasetRequest) -> Result<()> {
199 let embedding = request
201 .embedding
202 .get_or_insert_with(EmbeddingConfig::vectoramp);
203 if embedding.provider.is_none() {
204 embedding.provider = Some(DEFAULT_EMBEDDING_PROVIDER.into());
205 }
206 if embedding.model.is_none() {
207 embedding.model = Some(DEFAULT_EMBEDDING_MODEL.into());
208 }
209
210 if request.dim.is_none() {
212 let provider = request
213 .embedding
214 .as_ref()
215 .and_then(|e| e.provider.as_deref());
216 let model = request.embedding.as_ref().and_then(|e| e.model.as_deref());
217 match infer_embedding_dim(provider, model) {
218 Some(dim) => request.dim = Some(dim),
219 None => {
220 return Err(Error::invalid_input(format!(
221 "could not infer dim for embedding model {:?}; pass dim explicitly via CreateDatasetRequest::builder(name).dim(n)",
222 model.unwrap_or(DEFAULT_EMBEDDING_MODEL)
223 )));
224 }
225 }
226 }
227
228 if request.metric.is_none() {
229 request.metric = Some("cosine".into());
230 }
231 Ok(())
232 }
233
234 pub async fn delete(&self, dataset_id: &str) -> Result<()> {
236 self.client
237 .dispatcher()
238 .empty(Request {
239 method: "DELETE".into(),
240 path: format!("/datasets/{dataset_id}"),
241 ..Default::default()
242 })
243 .await
244 }
245
246 pub async fn patch_metadata_schema(
248 &self,
249 dataset_id: &str,
250 schema: Vec<MetadataSchemaField>,
251 ) -> Result<Dataset> {
252 self.update_metadata_schema(dataset_id, schema, MetadataSchemaUpdateMode::Merge)
253 .await
254 }
255
256 pub async fn replace_metadata_schema(
258 &self,
259 dataset_id: &str,
260 schema: Vec<MetadataSchemaField>,
261 ) -> Result<Dataset> {
262 self.update_metadata_schema(dataset_id, schema, MetadataSchemaUpdateMode::Replace)
263 .await
264 }
265
266 async fn update_metadata_schema(
267 &self,
268 dataset_id: &str,
269 schema: Vec<MetadataSchemaField>,
270 mode: MetadataSchemaUpdateMode,
271 ) -> Result<Dataset> {
272 let info: DatasetInfo = self
273 .client
274 .dispatcher()
275 .json(Request {
276 method: "PATCH".into(),
277 path: format!("/datasets/{dataset_id}/schema"),
278 body: Some(json!({ "schema": schema, "mode": mode })),
279 ..Default::default()
280 })
281 .await?;
282 Ok(Dataset::new(self.client.clone(), info))
283 }
284
285 pub async fn list_documents(
287 &self,
288 dataset_id: &str,
289 opts: DocumentListOpts,
290 ) -> Result<DatasetDocumentList> {
291 let mut req = Request {
292 method: "GET".into(),
293 path: format!("/datasets/{dataset_id}/documents"),
294 ..Default::default()
295 };
296 if let Some(limit) = opts.limit {
297 if limit > 0 {
298 req.query.push(("limit".into(), limit.to_string()));
299 }
300 }
301 if let Some(cursor) = opts.cursor {
302 if !cursor.is_empty() {
303 req.query.push(("cursor".into(), cursor));
304 }
305 }
306 if let Some(status) = opts.status {
307 if !status.is_empty() {
308 req.query.push(("status".into(), status));
309 }
310 }
311 self.client.dispatcher().json(req).await
312 }
313
314 pub async fn download_document(&self, dataset_id: &str, document_id: &str) -> Result<Vec<u8>> {
316 self.client
317 .dispatcher()
318 .bytes(Request {
319 method: "GET".into(),
320 path: format!("/datasets/{dataset_id}/documents/{document_id}/download"),
321 ..Default::default()
322 })
323 .await
324 }
325
326 pub async fn search<I: Into<SearchInput>>(
328 &self,
329 dataset_id: &str,
330 input: I,
331 ) -> Result<SearchResponse> {
332 self.search_with(dataset_id, input, SearchOptions::default())
333 .await
334 }
335
336 pub async fn search_with<I: Into<SearchInput>>(
338 &self,
339 dataset_id: &str,
340 input: I,
341 options: SearchOptions,
342 ) -> Result<SearchResponse> {
343 let mut req = match input.into() {
344 SearchInput::Text(text) => SearchRequest {
345 query_text: Some(text),
346 top_k: 0,
347 ..Default::default()
348 },
349 SearchInput::Vector(values) => SearchRequest {
350 query: Some(values),
351 top_k: 0,
352 ..Default::default()
353 },
354 SearchInput::Request(r) => *r,
355 };
356 if let Some(top_k) = options.top_k {
357 req.top_k = top_k;
358 }
359 if req.top_k == 0 {
360 req.top_k = DEFAULT_SEARCH_TOP_K;
361 }
362 if let Some(im) = options.include_metadata {
363 req.include_metadata = Some(im);
364 }
365 if let Some(id) = options.include_documents {
366 req.include_documents = Some(id);
367 }
368 if let Some(filters) = options.filters {
369 req.filters = Some(filters);
370 }
371 if let Some(rerank) = options.rerank {
372 req.rerank = Some(rerank);
373 }
374 let body = serde_json::to_value(&req)?;
375 self.client
376 .dispatcher()
377 .json(Request {
378 method: "POST".into(),
379 path: format!("/datasets/{dataset_id}/search"),
380 body: Some(body),
381 ..Default::default()
382 })
383 .await
384 }
385
386 pub async fn insert(
391 &self,
392 dataset_id: &str,
393 vectors: Vec<Vector>,
394 ) -> Result<InsertVectorsResponse> {
395 let body = serde_json::to_value(InsertVectorsRequest { vectors })?;
396 self.client
397 .dispatcher()
398 .json(Request {
399 method: "POST".into(),
400 path: format!("/datasets/{dataset_id}/insert"),
401 body: Some(body),
402 ..Default::default()
403 })
404 .await
405 }
406
407 pub async fn insert_vectors(
410 &self,
411 dataset_id: &str,
412 vectors: Vec<Vector>,
413 ) -> Result<InsertVectorsResponse> {
414 self.insert(dataset_id, vectors).await
415 }
416
417 pub async fn delete_vectors<I: Into<VectorId>>(
419 &self,
420 dataset_id: &str,
421 ids: Vec<I>,
422 ) -> Result<DeleteVectorsResponse> {
423 self.delete_vectors_with_write_concern(dataset_id, ids, None::<String>)
424 .await
425 }
426
427 pub async fn delete_vectors_with_write_concern<I: Into<VectorId>, S: Into<String>>(
429 &self,
430 dataset_id: &str,
431 ids: Vec<I>,
432 write_concern: Option<S>,
433 ) -> Result<DeleteVectorsResponse> {
434 let body = serde_json::to_value(DeleteVectorsRequest {
435 ids: ids.into_iter().map(Into::into).collect(),
436 write_concern: write_concern.map(Into::into),
437 })?;
438 self.client
439 .dispatcher()
440 .json(Request {
441 method: "DELETE".into(),
442 path: format!("/datasets/{dataset_id}/vectors"),
443 body: Some(body),
444 ..Default::default()
445 })
446 .await
447 }
448
449 pub async fn embed(&self, dataset_id: &str, request: EmbedRequest) -> Result<EmbedResponse> {
451 let body = serde_json::to_value(&request)?;
452 self.client
453 .dispatcher()
454 .json(Request {
455 method: "POST".into(),
456 path: format!("/datasets/{dataset_id}/embed"),
457 body: Some(body),
458 ..Default::default()
459 })
460 .await
461 }
462
463 pub async fn add_texts<I: Into<AddTextsInput>>(
466 &self,
467 dataset_id: &str,
468 input: I,
469 ) -> Result<AddTextsResponse> {
470 self.add_texts_with(dataset_id, input, AddTextsOptions::default())
471 .await
472 }
473
474 pub async fn add_texts_with<I: Into<AddTextsInput>>(
476 &self,
477 dataset_id: &str,
478 input: I,
479 options: AddTextsOptions,
480 ) -> Result<AddTextsResponse> {
481 let docs = match input.into() {
482 AddTextsInput::Texts(texts) => texts
483 .into_iter()
484 .enumerate()
485 .map(|(i, text)| TextDocument {
486 id: VectorId::Str(format!("text-{}", i + 1)),
487 text,
488 metadata: None,
489 })
490 .collect::<Vec<_>>(),
491 AddTextsInput::Documents(docs) => docs,
492 };
493
494 if docs.is_empty() {
495 return Ok(AddTextsResponse::default());
496 }
497
498 let texts: Vec<String> = docs.iter().map(|d| d.text.clone()).collect();
499 let embed_req = EmbedRequest {
500 texts: Some(texts),
501 embedding_provider: options.embedding_provider.clone(),
502 embedding_model: options.embedding_model.clone(),
503 ..Default::default()
504 };
505 let embed_resp = self.embed(dataset_id, embed_req).await?;
506 let mut embeddings = embed_resp.embeddings;
507 if embeddings.is_empty() {
508 if let Some(single) = embed_resp.embedding {
509 embeddings.push(single);
510 }
511 }
512
513 let vectors: Vec<Vector> = docs
514 .into_iter()
515 .enumerate()
516 .map(|(i, doc)| {
517 let mut metadata = doc.metadata.unwrap_or_default();
518 metadata
519 .entry("text".to_string())
520 .or_insert(Value::String(doc.text));
521 let values = embeddings.get(i).cloned().unwrap_or_default();
522 Vector {
523 id: doc.id,
524 values,
525 metadata: Some(metadata),
526 }
527 })
528 .collect();
529
530 let inserted = self.insert(dataset_id, vectors).await?;
531 Ok(AddTextsResponse {
532 inserted: inserted.inserted,
533 embeddings: embeddings.len() as u32,
534 })
535 }
536
537 pub async fn ingest_files(
539 &self,
540 dataset_id: &str,
541 paths: Vec<PathBuf>,
542 options: Option<IngestFilesOptions>,
543 ) -> Result<Job> {
544 self.client
545 .ingestion()
546 .ingest_files(dataset_id, paths, options)
547 .await
548 }
549
550 pub async fn ingest_source(&self, dataset_id: &str, source_id: &str) -> Result<Job> {
552 self.client
553 .ingestion()
554 .start_job(crate::types::StartIngestionRequest {
555 source_id: source_id.into(),
556 dataset_id: dataset_id.into(),
557 pipeline_id: None,
558 })
559 .await
560 }
561
562 pub async fn ingest_source_with_pipeline(
564 &self,
565 dataset_id: &str,
566 source_id: &str,
567 pipeline_id: &str,
568 ) -> Result<Job> {
569 self.client
570 .ingestion()
571 .start_job(crate::types::StartIngestionRequest {
572 source_id: source_id.into(),
573 dataset_id: dataset_id.into(),
574 pipeline_id: Some(pipeline_id.into()),
575 })
576 .await
577 }
578
579 pub async fn ingest_new_source<B: IntoCreateSourceRequest>(
582 &self,
583 dataset_id: &str,
584 builder: B,
585 ) -> Result<Job> {
586 let source = self.client.ingestion().create_source(builder).await?;
587 let id = source
588 .identifier()
589 .ok_or_else(|| {
590 Error::invalid_input("source response did not include id, source_id, or uuid")
591 })?
592 .to_owned();
593 self.ingest_source(dataset_id, &id).await
594 }
595
596 pub async fn ask<S: Into<String>>(&self, dataset_id: &str, query: S) -> Result<AskResponse> {
598 self.ask_with(dataset_id, query, AskOptions::default())
599 .await
600 }
601
602 pub async fn ask_with<S: Into<String>>(
604 &self,
605 dataset_id: &str,
606 query: S,
607 mut options: AskOptions,
608 ) -> Result<AskResponse> {
609 options.dataset_ids = Some(vec![dataset_id.to_owned()]);
610 self.client
611 .intelligence()
612 .ask_with(query.into(), options)
613 .await
614 }
615
616 pub async fn ask_stream<S: Into<String>>(
618 &self,
619 dataset_id: &str,
620 query: S,
621 ) -> Result<crate::intelligence::AskStream> {
622 let options = AskOptions::default().with_dataset(dataset_id.to_owned());
623 self.client
624 .intelligence()
625 .stream(query.into(), options)
626 .await
627 }
628}
629
630pub enum SearchInput {
632 Text(String),
633 Vector(Vec<f64>),
634 Request(Box<SearchRequest>),
635}
636
637impl From<&str> for SearchInput {
638 fn from(value: &str) -> Self {
639 SearchInput::Text(value.to_owned())
640 }
641}
642
643impl From<String> for SearchInput {
644 fn from(value: String) -> Self {
645 SearchInput::Text(value)
646 }
647}
648
649impl From<Vec<f64>> for SearchInput {
650 fn from(value: Vec<f64>) -> Self {
651 SearchInput::Vector(value)
652 }
653}
654
655impl From<SearchRequest> for SearchInput {
656 fn from(value: SearchRequest) -> Self {
657 SearchInput::Request(Box::new(value))
658 }
659}
660
661#[derive(Debug, Default, Clone)]
663pub struct SearchOptions {
664 pub top_k: Option<u32>,
665 pub include_metadata: Option<bool>,
666 pub include_documents: Option<bool>,
667 pub filters: Option<HashMap<String, String>>,
668 pub rerank: Option<Rerank>,
669}
670
671impl SearchOptions {
672 pub fn with_rerank(mut self, enabled: bool) -> Self {
674 self.rerank = Some(Rerank::Enabled(enabled));
675 self
676 }
677
678 pub fn with_rerank_config(mut self, config: RerankConfig) -> Self {
680 self.rerank = Some(Rerank::Config(config));
681 self
682 }
683}
684
685pub enum AddTextsInput {
687 Texts(Vec<String>),
688 Documents(Vec<TextDocument>),
689}
690
691impl From<&str> for AddTextsInput {
692 fn from(v: &str) -> Self {
693 AddTextsInput::Texts(vec![v.to_owned()])
694 }
695}
696
697impl From<String> for AddTextsInput {
698 fn from(v: String) -> Self {
699 AddTextsInput::Texts(vec![v])
700 }
701}
702
703impl From<Vec<String>> for AddTextsInput {
704 fn from(v: Vec<String>) -> Self {
705 AddTextsInput::Texts(v)
706 }
707}
708
709impl From<Vec<&str>> for AddTextsInput {
710 fn from(v: Vec<&str>) -> Self {
711 AddTextsInput::Texts(v.into_iter().map(|s| s.to_owned()).collect())
712 }
713}
714
715impl From<Vec<TextDocument>> for AddTextsInput {
716 fn from(v: Vec<TextDocument>) -> Self {
717 AddTextsInput::Documents(v)
718 }
719}
720
721#[derive(Debug, Default, Clone)]
723pub struct AddTextsOptions {
724 pub embedding_provider: Option<String>,
725 pub embedding_model: Option<String>,
726}
727
728#[derive(Clone)]
731pub struct Dataset {
732 client: Client,
733 pub info: DatasetInfo,
734}
735
736impl Dataset {
737 pub(crate) fn new(client: Client, info: DatasetInfo) -> Self {
738 Self { client, info }
739 }
740
741 pub fn id(&self) -> &str {
743 &self.info.id
744 }
745
746 pub async fn search<I: Into<SearchInput>>(&self, input: I) -> Result<SearchResponse> {
748 self.client.datasets().search(self.id(), input).await
749 }
750
751 pub async fn search_with<I: Into<SearchInput>>(
753 &self,
754 input: I,
755 options: SearchOptions,
756 ) -> Result<SearchResponse> {
757 self.client
758 .datasets()
759 .search_with(self.id(), input, options)
760 .await
761 }
762
763 pub async fn insert(&self, vectors: Vec<Vector>) -> Result<InsertVectorsResponse> {
765 self.client.datasets().insert(self.id(), vectors).await
766 }
767
768 pub async fn insert_vectors(&self, vectors: Vec<Vector>) -> Result<InsertVectorsResponse> {
771 self.client.datasets().insert(self.id(), vectors).await
772 }
773
774 pub async fn delete_vectors<I: Into<VectorId>>(
776 &self,
777 ids: Vec<I>,
778 ) -> Result<DeleteVectorsResponse> {
779 self.client.datasets().delete_vectors(self.id(), ids).await
780 }
781
782 pub async fn delete_vectors_with_write_concern<I: Into<VectorId>, S: Into<String>>(
784 &self,
785 ids: Vec<I>,
786 write_concern: Option<S>,
787 ) -> Result<DeleteVectorsResponse> {
788 self.client
789 .datasets()
790 .delete_vectors_with_write_concern(self.id(), ids, write_concern)
791 .await
792 }
793
794 pub async fn add_texts<I: Into<AddTextsInput>>(&self, input: I) -> Result<AddTextsResponse> {
796 self.client.datasets().add_texts(self.id(), input).await
797 }
798
799 pub async fn add_texts_with<I: Into<AddTextsInput>>(
801 &self,
802 input: I,
803 options: AddTextsOptions,
804 ) -> Result<AddTextsResponse> {
805 self.client
806 .datasets()
807 .add_texts_with(self.id(), input, options)
808 .await
809 }
810
811 pub async fn embed(&self, request: EmbedRequest) -> Result<EmbedResponse> {
813 self.client.datasets().embed(self.id(), request).await
814 }
815
816 pub async fn ask<S: Into<String>>(&self, query: S) -> Result<AskResponse> {
818 self.client.datasets().ask(self.id(), query).await
819 }
820
821 pub async fn ask_with<S: Into<String>>(
823 &self,
824 query: S,
825 options: AskOptions,
826 ) -> Result<AskResponse> {
827 self.client
828 .datasets()
829 .ask_with(self.id(), query, options)
830 .await
831 }
832
833 pub async fn ask_stream<S: Into<String>>(
835 &self,
836 query: S,
837 ) -> Result<crate::intelligence::AskStream> {
838 self.client.datasets().ask_stream(self.id(), query).await
839 }
840
841 pub async fn list_documents(&self, opts: DocumentListOpts) -> Result<DatasetDocumentList> {
843 self.client.datasets().list_documents(self.id(), opts).await
844 }
845
846 pub async fn download_document(&self, document_id: &str) -> Result<Vec<u8>> {
848 self.client
849 .datasets()
850 .download_document(self.id(), document_id)
851 .await
852 }
853
854 pub async fn ingest_files(
856 &self,
857 paths: Vec<PathBuf>,
858 options: Option<IngestFilesOptions>,
859 ) -> Result<Job> {
860 self.client
861 .datasets()
862 .ingest_files(self.id(), paths, options)
863 .await
864 }
865
866 pub async fn ingest_source(&self, source_id: &str) -> Result<Job> {
868 self.client
869 .datasets()
870 .ingest_source(self.id(), source_id)
871 .await
872 }
873
874 pub async fn ingest_source_with_pipeline(
876 &self,
877 source_id: &str,
878 pipeline_id: &str,
879 ) -> Result<Job> {
880 self.client
881 .datasets()
882 .ingest_source_with_pipeline(self.id(), source_id, pipeline_id)
883 .await
884 }
885
886 pub async fn ingest_new_source<B: IntoCreateSourceRequest>(&self, builder: B) -> Result<Job> {
888 self.client
889 .datasets()
890 .ingest_new_source(self.id(), builder)
891 .await
892 }
893
894 pub async fn delete(self) -> Result<()> {
896 self.client.datasets().delete(&self.info.id).await
897 }
898}
899
900pub fn metadata() -> Metadata {
902 Metadata::new()
903}
904
905pub(crate) fn push_pagination(query: &mut Vec<(String, String)>, limit: u32, offset: u32) {
906 if limit > 0 {
907 query.push(("limit".into(), limit.to_string()));
908 }
909 if offset > 0 {
910 query.push(("offset".into(), offset.to_string()));
911 }
912}