use std::collections::HashMap;
use std::path::PathBuf;
use serde_json::{json, Value};
use crate::client::Client;
use crate::errors::{Error, Result};
use crate::ingestion::IngestFilesOptions;
use crate::intelligence::AskOptions;
use crate::secrets::OPENAI_API_KEY_SECRET_REF;
use crate::sources::IntoCreateSourceRequest;
use crate::transport::Request;
use crate::types::{
infer_embedding_dim, AddTextsResponse, AskResponse, CreateDatasetRequest, DatasetDocumentList,
DatasetInfo, DatasetList, DeleteVectorsRequest, DeleteVectorsResponse, DocumentListOpts,
EmbedRequest, EmbedResponse, EmbeddingConfig, InsertVectorsRequest, InsertVectorsResponse, Job,
Metadata, MetadataSchemaField, MetadataSchemaUpdateMode, Rerank, RerankConfig, SearchRequest,
SearchResponse, TextDocument, Vector, VectorId, DEFAULT_EMBEDDING_MODEL,
DEFAULT_EMBEDDING_PROVIDER,
};
pub const DEFAULT_SEARCH_TOP_K: u32 = 10;
pub const DEFAULT_PAGE_LIMIT: u32 = 50;
#[derive(Debug, Clone, Copy, Default)]
pub struct Pagination {
pub limit: Option<u32>,
pub offset: Option<u32>,
}
impl Pagination {
pub(crate) fn resolve(self) -> (u32, u32) {
(
self.limit.unwrap_or(DEFAULT_PAGE_LIMIT),
self.offset.unwrap_or(0),
)
}
}
impl From<()> for Pagination {
fn from(_: ()) -> Self {
Pagination::default()
}
}
impl From<u32> for Pagination {
fn from(limit: u32) -> Self {
Pagination {
limit: Some(limit),
offset: None,
}
}
}
impl From<(u32, u32)> for Pagination {
fn from((limit, offset): (u32, u32)) -> Self {
Pagination {
limit: Some(limit),
offset: Some(offset),
}
}
}
#[derive(Clone)]
pub struct DatasetService {
client: Client,
}
impl DatasetService {
pub(crate) fn new(client: Client) -> Self {
Self { client }
}
pub async fn list<P: Into<Pagination>>(&self, pagination: P) -> Result<DatasetList> {
let (limit, offset) = pagination.into().resolve();
let mut req = Request {
method: "GET".into(),
path: "/datasets".into(),
..Default::default()
};
push_pagination(&mut req.query, limit, offset);
self.client.dispatcher().json(req).await
}
pub async fn get(&self, dataset_id: &str) -> Result<Dataset> {
let info: DatasetInfo = self
.client
.dispatcher()
.json(Request {
method: "GET".into(),
path: format!("/datasets/{dataset_id}"),
..Default::default()
})
.await?;
Ok(Dataset::new(self.client.clone(), info))
}
pub async fn create<R: Into<CreateDatasetRequest>>(&self, request: R) -> Result<Dataset> {
let mut request = request.into();
self.apply_create_defaults(&mut request)?;
let mut body = serde_json::to_value(&request)?;
if let Value::Object(ref mut map) = body {
map.insert("index_type".into(), Value::String("sable".into()));
}
let info: DatasetInfo = self
.client
.dispatcher()
.json(Request {
method: "POST".into(),
path: "/datasets".into(),
body: Some(body),
..Default::default()
})
.await?;
Ok(Dataset::new(self.client.clone(), info))
}
pub async fn create_with_openai_api_key<R: Into<CreateDatasetRequest>, S: Into<String>>(
&self,
request: R,
api_key: S,
) -> Result<Dataset> {
self.client
.org_secrets()
.put_openai_api_key(api_key.into())
.await?;
let mut request = request.into();
let embedding = request
.embedding
.get_or_insert_with(|| EmbeddingConfig::openai("small"));
embedding.provider = Some("openai".into());
if embedding.model.is_none() {
embedding.model = Some("text-embedding-3-small".into());
}
embedding.secret_ref = Some(OPENAI_API_KEY_SECRET_REF.into());
self.create(request).await
}
fn apply_create_defaults(&self, request: &mut CreateDatasetRequest) -> Result<()> {
let embedding = request
.embedding
.get_or_insert_with(EmbeddingConfig::vectoramp);
if embedding.provider.is_none() {
embedding.provider = Some(DEFAULT_EMBEDDING_PROVIDER.into());
}
if embedding.model.is_none() {
embedding.model = Some(DEFAULT_EMBEDDING_MODEL.into());
}
if request.dim.is_none() {
let provider = request
.embedding
.as_ref()
.and_then(|e| e.provider.as_deref());
let model = request.embedding.as_ref().and_then(|e| e.model.as_deref());
match infer_embedding_dim(provider, model) {
Some(dim) => request.dim = Some(dim),
None => {
return Err(Error::invalid_input(format!(
"could not infer dim for embedding model {:?}; pass dim explicitly via CreateDatasetRequest::builder(name).dim(n)",
model.unwrap_or(DEFAULT_EMBEDDING_MODEL)
)));
}
}
}
if request.metric.is_none() {
request.metric = Some("cosine".into());
}
Ok(())
}
pub async fn delete(&self, dataset_id: &str) -> Result<()> {
self.client
.dispatcher()
.empty(Request {
method: "DELETE".into(),
path: format!("/datasets/{dataset_id}"),
..Default::default()
})
.await
}
pub async fn patch_metadata_schema(
&self,
dataset_id: &str,
schema: Vec<MetadataSchemaField>,
) -> Result<Dataset> {
self.update_metadata_schema(dataset_id, schema, MetadataSchemaUpdateMode::Merge)
.await
}
pub async fn replace_metadata_schema(
&self,
dataset_id: &str,
schema: Vec<MetadataSchemaField>,
) -> Result<Dataset> {
self.update_metadata_schema(dataset_id, schema, MetadataSchemaUpdateMode::Replace)
.await
}
async fn update_metadata_schema(
&self,
dataset_id: &str,
schema: Vec<MetadataSchemaField>,
mode: MetadataSchemaUpdateMode,
) -> Result<Dataset> {
let info: DatasetInfo = self
.client
.dispatcher()
.json(Request {
method: "PATCH".into(),
path: format!("/datasets/{dataset_id}/schema"),
body: Some(json!({ "schema": schema, "mode": mode })),
..Default::default()
})
.await?;
Ok(Dataset::new(self.client.clone(), info))
}
pub async fn list_documents(
&self,
dataset_id: &str,
opts: DocumentListOpts,
) -> Result<DatasetDocumentList> {
let mut req = Request {
method: "GET".into(),
path: format!("/datasets/{dataset_id}/documents"),
..Default::default()
};
if let Some(limit) = opts.limit {
if limit > 0 {
req.query.push(("limit".into(), limit.to_string()));
}
}
if let Some(cursor) = opts.cursor {
if !cursor.is_empty() {
req.query.push(("cursor".into(), cursor));
}
}
if let Some(status) = opts.status {
if !status.is_empty() {
req.query.push(("status".into(), status));
}
}
self.client.dispatcher().json(req).await
}
pub async fn download_document(&self, dataset_id: &str, document_id: &str) -> Result<Vec<u8>> {
self.client
.dispatcher()
.bytes(Request {
method: "GET".into(),
path: format!("/datasets/{dataset_id}/documents/{document_id}/download"),
..Default::default()
})
.await
}
pub async fn search<I: Into<SearchInput>>(
&self,
dataset_id: &str,
input: I,
) -> Result<SearchResponse> {
self.search_with(dataset_id, input, SearchOptions::default())
.await
}
pub async fn search_with<I: Into<SearchInput>>(
&self,
dataset_id: &str,
input: I,
options: SearchOptions,
) -> Result<SearchResponse> {
let mut req = match input.into() {
SearchInput::Text(text) => SearchRequest {
query_text: Some(text),
top_k: 0,
..Default::default()
},
SearchInput::Vector(values) => SearchRequest {
query: Some(values),
top_k: 0,
..Default::default()
},
SearchInput::Request(r) => *r,
};
if let Some(top_k) = options.top_k {
req.top_k = top_k;
}
if req.top_k == 0 {
req.top_k = DEFAULT_SEARCH_TOP_K;
}
if let Some(im) = options.include_metadata {
req.include_metadata = Some(im);
}
if let Some(id) = options.include_documents {
req.include_documents = Some(id);
}
if let Some(filters) = options.filters {
req.filters = Some(filters);
}
if let Some(rerank) = options.rerank {
req.rerank = Some(rerank);
}
let body = serde_json::to_value(&req)?;
self.client
.dispatcher()
.json(Request {
method: "POST".into(),
path: format!("/datasets/{dataset_id}/search"),
body: Some(body),
..Default::default()
})
.await
}
pub async fn insert(
&self,
dataset_id: &str,
vectors: Vec<Vector>,
) -> Result<InsertVectorsResponse> {
let body = serde_json::to_value(InsertVectorsRequest { vectors })?;
self.client
.dispatcher()
.json(Request {
method: "POST".into(),
path: format!("/datasets/{dataset_id}/insert"),
body: Some(body),
..Default::default()
})
.await
}
pub async fn insert_vectors(
&self,
dataset_id: &str,
vectors: Vec<Vector>,
) -> Result<InsertVectorsResponse> {
self.insert(dataset_id, vectors).await
}
pub async fn delete_vectors<I: Into<VectorId>>(
&self,
dataset_id: &str,
ids: Vec<I>,
) -> Result<DeleteVectorsResponse> {
self.delete_vectors_with_write_concern(dataset_id, ids, None::<String>)
.await
}
pub async fn delete_vectors_with_write_concern<I: Into<VectorId>, S: Into<String>>(
&self,
dataset_id: &str,
ids: Vec<I>,
write_concern: Option<S>,
) -> Result<DeleteVectorsResponse> {
let body = serde_json::to_value(DeleteVectorsRequest {
ids: ids.into_iter().map(Into::into).collect(),
write_concern: write_concern.map(Into::into),
})?;
self.client
.dispatcher()
.json(Request {
method: "DELETE".into(),
path: format!("/datasets/{dataset_id}/vectors"),
body: Some(body),
..Default::default()
})
.await
}
pub async fn embed(&self, dataset_id: &str, request: EmbedRequest) -> Result<EmbedResponse> {
let body = serde_json::to_value(&request)?;
self.client
.dispatcher()
.json(Request {
method: "POST".into(),
path: format!("/datasets/{dataset_id}/embed"),
body: Some(body),
..Default::default()
})
.await
}
pub async fn add_texts<I: Into<AddTextsInput>>(
&self,
dataset_id: &str,
input: I,
) -> Result<AddTextsResponse> {
self.add_texts_with(dataset_id, input, AddTextsOptions::default())
.await
}
pub async fn add_texts_with<I: Into<AddTextsInput>>(
&self,
dataset_id: &str,
input: I,
options: AddTextsOptions,
) -> Result<AddTextsResponse> {
let docs = match input.into() {
AddTextsInput::Texts(texts) => texts
.into_iter()
.enumerate()
.map(|(i, text)| TextDocument {
id: VectorId::Str(format!("text-{}", i + 1)),
text,
metadata: None,
})
.collect::<Vec<_>>(),
AddTextsInput::Documents(docs) => docs,
};
if docs.is_empty() {
return Ok(AddTextsResponse::default());
}
let texts: Vec<String> = docs.iter().map(|d| d.text.clone()).collect();
let embed_req = EmbedRequest {
texts: Some(texts),
embedding_provider: options.embedding_provider.clone(),
embedding_model: options.embedding_model.clone(),
..Default::default()
};
let embed_resp = self.embed(dataset_id, embed_req).await?;
let mut embeddings = embed_resp.embeddings;
if embeddings.is_empty() {
if let Some(single) = embed_resp.embedding {
embeddings.push(single);
}
}
let vectors: Vec<Vector> = docs
.into_iter()
.enumerate()
.map(|(i, doc)| {
let mut metadata = doc.metadata.unwrap_or_default();
metadata
.entry("text".to_string())
.or_insert(Value::String(doc.text));
let values = embeddings.get(i).cloned().unwrap_or_default();
Vector {
id: doc.id,
values,
metadata: Some(metadata),
}
})
.collect();
let inserted = self.insert(dataset_id, vectors).await?;
Ok(AddTextsResponse {
inserted: inserted.inserted,
embeddings: embeddings.len() as u32,
})
}
pub async fn ingest_files(
&self,
dataset_id: &str,
paths: Vec<PathBuf>,
options: Option<IngestFilesOptions>,
) -> Result<Job> {
self.client
.ingestion()
.ingest_files(dataset_id, paths, options)
.await
}
pub async fn ingest_source(&self, dataset_id: &str, source_id: &str) -> Result<Job> {
self.client
.ingestion()
.start_job(crate::types::StartIngestionRequest {
source_id: source_id.into(),
dataset_id: dataset_id.into(),
pipeline_id: None,
})
.await
}
pub async fn ingest_source_with_pipeline(
&self,
dataset_id: &str,
source_id: &str,
pipeline_id: &str,
) -> Result<Job> {
self.client
.ingestion()
.start_job(crate::types::StartIngestionRequest {
source_id: source_id.into(),
dataset_id: dataset_id.into(),
pipeline_id: Some(pipeline_id.into()),
})
.await
}
pub async fn ingest_new_source<B: IntoCreateSourceRequest>(
&self,
dataset_id: &str,
builder: B,
) -> Result<Job> {
let source = self.client.ingestion().create_source(builder).await?;
let id = source
.identifier()
.ok_or_else(|| {
Error::invalid_input("source response did not include id, source_id, or uuid")
})?
.to_owned();
self.ingest_source(dataset_id, &id).await
}
pub async fn ask<S: Into<String>>(&self, dataset_id: &str, query: S) -> Result<AskResponse> {
self.ask_with(dataset_id, query, AskOptions::default())
.await
}
pub async fn ask_with<S: Into<String>>(
&self,
dataset_id: &str,
query: S,
mut options: AskOptions,
) -> Result<AskResponse> {
options.dataset_id = Some(json!(dataset_id.to_owned()));
self.client
.intelligence()
.ask_with(query.into(), options)
.await
}
pub async fn ask_stream<S: Into<String>>(
&self,
dataset_id: &str,
query: S,
) -> Result<crate::intelligence::AskStream> {
let options = AskOptions::default().with_dataset(dataset_id.to_owned());
self.client
.intelligence()
.stream(query.into(), options)
.await
}
}
pub enum SearchInput {
Text(String),
Vector(Vec<f64>),
Request(Box<SearchRequest>),
}
impl From<&str> for SearchInput {
fn from(value: &str) -> Self {
SearchInput::Text(value.to_owned())
}
}
impl From<String> for SearchInput {
fn from(value: String) -> Self {
SearchInput::Text(value)
}
}
impl From<Vec<f64>> for SearchInput {
fn from(value: Vec<f64>) -> Self {
SearchInput::Vector(value)
}
}
impl From<SearchRequest> for SearchInput {
fn from(value: SearchRequest) -> Self {
SearchInput::Request(Box::new(value))
}
}
#[derive(Debug, Default, Clone)]
pub struct SearchOptions {
pub top_k: Option<u32>,
pub include_metadata: Option<bool>,
pub include_documents: Option<bool>,
pub filters: Option<HashMap<String, String>>,
pub rerank: Option<Rerank>,
}
impl SearchOptions {
pub fn with_rerank(mut self, enabled: bool) -> Self {
self.rerank = Some(Rerank::Enabled(enabled));
self
}
pub fn with_rerank_config(mut self, config: RerankConfig) -> Self {
self.rerank = Some(Rerank::Config(config));
self
}
}
pub enum AddTextsInput {
Texts(Vec<String>),
Documents(Vec<TextDocument>),
}
impl From<&str> for AddTextsInput {
fn from(v: &str) -> Self {
AddTextsInput::Texts(vec![v.to_owned()])
}
}
impl From<String> for AddTextsInput {
fn from(v: String) -> Self {
AddTextsInput::Texts(vec![v])
}
}
impl From<Vec<String>> for AddTextsInput {
fn from(v: Vec<String>) -> Self {
AddTextsInput::Texts(v)
}
}
impl From<Vec<&str>> for AddTextsInput {
fn from(v: Vec<&str>) -> Self {
AddTextsInput::Texts(v.into_iter().map(|s| s.to_owned()).collect())
}
}
impl From<Vec<TextDocument>> for AddTextsInput {
fn from(v: Vec<TextDocument>) -> Self {
AddTextsInput::Documents(v)
}
}
#[derive(Debug, Default, Clone)]
pub struct AddTextsOptions {
pub embedding_provider: Option<String>,
pub embedding_model: Option<String>,
}
#[derive(Clone)]
pub struct Dataset {
client: Client,
pub info: DatasetInfo,
}
impl Dataset {
pub(crate) fn new(client: Client, info: DatasetInfo) -> Self {
Self { client, info }
}
pub fn id(&self) -> &str {
&self.info.id
}
pub async fn search<I: Into<SearchInput>>(&self, input: I) -> Result<SearchResponse> {
self.client.datasets().search(self.id(), input).await
}
pub async fn search_with<I: Into<SearchInput>>(
&self,
input: I,
options: SearchOptions,
) -> Result<SearchResponse> {
self.client
.datasets()
.search_with(self.id(), input, options)
.await
}
pub async fn insert(&self, vectors: Vec<Vector>) -> Result<InsertVectorsResponse> {
self.client.datasets().insert(self.id(), vectors).await
}
pub async fn insert_vectors(&self, vectors: Vec<Vector>) -> Result<InsertVectorsResponse> {
self.client.datasets().insert(self.id(), vectors).await
}
pub async fn delete_vectors<I: Into<VectorId>>(
&self,
ids: Vec<I>,
) -> Result<DeleteVectorsResponse> {
self.client.datasets().delete_vectors(self.id(), ids).await
}
pub async fn delete_vectors_with_write_concern<I: Into<VectorId>, S: Into<String>>(
&self,
ids: Vec<I>,
write_concern: Option<S>,
) -> Result<DeleteVectorsResponse> {
self.client
.datasets()
.delete_vectors_with_write_concern(self.id(), ids, write_concern)
.await
}
pub async fn add_texts<I: Into<AddTextsInput>>(&self, input: I) -> Result<AddTextsResponse> {
self.client.datasets().add_texts(self.id(), input).await
}
pub async fn add_texts_with<I: Into<AddTextsInput>>(
&self,
input: I,
options: AddTextsOptions,
) -> Result<AddTextsResponse> {
self.client
.datasets()
.add_texts_with(self.id(), input, options)
.await
}
pub async fn embed(&self, request: EmbedRequest) -> Result<EmbedResponse> {
self.client.datasets().embed(self.id(), request).await
}
pub async fn ask<S: Into<String>>(&self, query: S) -> Result<AskResponse> {
self.client.datasets().ask(self.id(), query).await
}
pub async fn ask_with<S: Into<String>>(
&self,
query: S,
options: AskOptions,
) -> Result<AskResponse> {
self.client
.datasets()
.ask_with(self.id(), query, options)
.await
}
pub async fn ask_stream<S: Into<String>>(
&self,
query: S,
) -> Result<crate::intelligence::AskStream> {
self.client.datasets().ask_stream(self.id(), query).await
}
pub async fn list_documents(&self, opts: DocumentListOpts) -> Result<DatasetDocumentList> {
self.client.datasets().list_documents(self.id(), opts).await
}
pub async fn download_document(&self, document_id: &str) -> Result<Vec<u8>> {
self.client
.datasets()
.download_document(self.id(), document_id)
.await
}
pub async fn ingest_files(
&self,
paths: Vec<PathBuf>,
options: Option<IngestFilesOptions>,
) -> Result<Job> {
self.client
.datasets()
.ingest_files(self.id(), paths, options)
.await
}
pub async fn ingest_source(&self, source_id: &str) -> Result<Job> {
self.client
.datasets()
.ingest_source(self.id(), source_id)
.await
}
pub async fn ingest_source_with_pipeline(
&self,
source_id: &str,
pipeline_id: &str,
) -> Result<Job> {
self.client
.datasets()
.ingest_source_with_pipeline(self.id(), source_id, pipeline_id)
.await
}
pub async fn ingest_new_source<B: IntoCreateSourceRequest>(&self, builder: B) -> Result<Job> {
self.client
.datasets()
.ingest_new_source(self.id(), builder)
.await
}
pub async fn delete(self) -> Result<()> {
self.client.datasets().delete(&self.info.id).await
}
}
pub fn metadata() -> Metadata {
Metadata::new()
}
pub(crate) fn push_pagination(query: &mut Vec<(String, String)>, limit: u32, offset: u32) {
if limit > 0 {
query.push(("limit".into(), limit.to_string()));
}
if offset > 0 {
query.push(("offset".into(), offset.to_string()));
}
}