use std::collections::HashMap;
use serde::{Deserialize, Serialize};
use serde_json::Value;
pub type Metadata = HashMap<String, Value>;
pub const DEFAULT_EMBEDDING_PROVIDER: &str = "vectoramp";
pub const DEFAULT_EMBEDDING_MODEL: &str = "VectorAmp-Embedding-4B";
pub const DEFAULT_EMBEDDING_DIM: u32 = 2560;
#[derive(Debug, Default, Clone, Serialize, Deserialize)]
pub struct EmbeddingConfig {
#[serde(skip_serializing_if = "Option::is_none")]
pub provider: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub model: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub secret_ref: Option<String>,
}
impl EmbeddingConfig {
pub fn vectoramp() -> Self {
Self {
provider: Some(DEFAULT_EMBEDDING_PROVIDER.into()),
model: Some(DEFAULT_EMBEDDING_MODEL.into()),
secret_ref: None,
}
}
pub fn openai<S: Into<String>>(model: S) -> Self {
let model = match model.into().as_str() {
"small" => "text-embedding-3-small".to_string(),
"large" => "text-embedding-3-large".to_string(),
other => other.to_string(),
};
Self {
provider: Some("openai".into()),
model: Some(model),
secret_ref: None,
}
}
}
pub fn infer_embedding_dim(provider: Option<&str>, model: Option<&str>) -> Option<u32> {
match (provider, model) {
(_, Some(DEFAULT_EMBEDDING_MODEL)) => Some(DEFAULT_EMBEDDING_DIM),
(Some("openai"), Some("text-embedding-3-small")) => Some(1536),
(Some("openai"), Some("text-embedding-3-large")) => Some(3072),
(_, Some("text-embedding-3-small")) => Some(1536),
(_, Some("text-embedding-3-large")) => Some(3072),
_ => None,
}
}
#[derive(Debug, Default, Clone, Serialize, Deserialize)]
pub struct DatasetInfo {
pub id: String,
pub name: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub org_id: Option<String>,
pub dim: u32,
pub metric: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub tuning: Option<HashMap<String, Value>>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub embedding: Option<EmbeddingConfig>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub index_type: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub created_at: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub updated_at: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub metadata: Option<Metadata>,
}
#[derive(Debug, Default, Clone, Deserialize)]
pub struct DatasetList {
#[serde(default)]
pub datasets: Vec<DatasetInfo>,
#[serde(default)]
pub total: u32,
#[serde(default)]
pub limit: u32,
#[serde(default)]
pub offset: u32,
}
#[derive(Debug, Default, Clone, Serialize, Deserialize)]
pub struct CreateDatasetRequest {
pub name: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub dim: Option<u32>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub metric: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub tuning: Option<HashMap<String, Value>>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub embedding: Option<EmbeddingConfig>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub hybrid: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub metadata: Option<Metadata>,
}
impl CreateDatasetRequest {
pub fn builder<S: Into<String>>(name: S) -> CreateDatasetBuilder {
CreateDatasetBuilder::new(name)
}
}
#[derive(Debug, Clone)]
pub struct CreateDatasetBuilder {
request: CreateDatasetRequest,
}
impl CreateDatasetBuilder {
pub fn new<S: Into<String>>(name: S) -> Self {
Self {
request: CreateDatasetRequest {
name: name.into(),
..Default::default()
},
}
}
pub fn dim(mut self, dim: u32) -> Self {
self.request.dim = Some(dim);
self
}
pub fn metric<S: Into<String>>(mut self, metric: S) -> Self {
self.request.metric = Some(metric.into());
self
}
pub fn embedding(mut self, embedding: EmbeddingConfig) -> Self {
self.request.embedding = Some(embedding);
self
}
pub fn openai<S: Into<String>>(mut self, model: S) -> Self {
self.request.embedding = Some(EmbeddingConfig::openai(model));
self
}
pub fn hybrid(mut self, hybrid: bool) -> Self {
self.request.hybrid = Some(hybrid);
self
}
pub fn metadata(mut self, metadata: Metadata) -> Self {
self.request.metadata = Some(metadata);
self
}
pub fn build(self) -> CreateDatasetRequest {
self.request
}
}
impl From<CreateDatasetBuilder> for CreateDatasetRequest {
fn from(b: CreateDatasetBuilder) -> Self {
b.build()
}
}
impl From<&str> for CreateDatasetRequest {
fn from(name: &str) -> Self {
CreateDatasetRequest {
name: name.to_owned(),
..Default::default()
}
}
}
impl From<String> for CreateDatasetRequest {
fn from(name: String) -> Self {
CreateDatasetRequest {
name,
..Default::default()
}
}
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(untagged)]
pub enum VectorId {
Int(i64),
Str(String),
}
impl Default for VectorId {
fn default() -> Self {
VectorId::Str(String::new())
}
}
impl std::fmt::Display for VectorId {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
VectorId::Int(n) => write!(f, "{n}"),
VectorId::Str(s) => write!(f, "{s}"),
}
}
}
impl From<String> for VectorId {
fn from(s: String) -> Self {
VectorId::Str(s)
}
}
impl From<&str> for VectorId {
fn from(s: &str) -> Self {
VectorId::Str(s.to_owned())
}
}
macro_rules! vector_id_from_int {
($($t:ty),*) => {$(
impl From<$t> for VectorId {
fn from(n: $t) -> Self {
VectorId::Int(n as i64)
}
}
)*};
}
vector_id_from_int!(i8, i16, i32, i64, u8, u16, u32, u64, usize, isize);
#[derive(Debug, Default, Clone, Serialize, Deserialize)]
pub struct Vector {
pub id: VectorId,
pub values: Vec<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub metadata: Option<Metadata>,
}
impl Vector {
pub fn new<I: Into<VectorId>>(id: I, values: Vec<f64>) -> Self {
Self {
id: id.into(),
values,
metadata: None,
}
}
pub fn with_metadata(mut self, metadata: Metadata) -> Self {
self.metadata = Some(metadata);
self
}
}
#[derive(Debug, Default, Clone, Serialize)]
pub(crate) struct InsertVectorsRequest {
pub vectors: Vec<Vector>,
}
#[derive(Debug, Default, Clone, Deserialize)]
pub struct InsertVectorsResponse {
#[serde(default)]
pub inserted: u32,
}
#[derive(Debug, Default, Clone, Serialize)]
pub(crate) struct DeleteVectorsRequest {
pub ids: Vec<VectorId>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub write_concern: Option<String>,
}
#[derive(Debug, Default, Clone, Deserialize)]
pub struct DeleteVectorsResponse {
#[serde(default)]
pub deleted: u32,
#[serde(default)]
pub dataset_id: Option<String>,
}
#[derive(Debug, Default, Clone)]
pub struct TextDocument {
pub id: VectorId,
pub text: String,
pub metadata: Option<Metadata>,
}
#[derive(Debug, Default, Clone, Deserialize)]
pub struct AddTextsResponse {
#[serde(default)]
pub inserted: u32,
#[serde(default)]
pub embeddings: u32,
}
#[derive(Debug, Default, Clone, Serialize)]
pub struct EmbedRequest {
#[serde(skip_serializing_if = "Option::is_none")]
pub text: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub texts: Option<Vec<String>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub embedding_provider: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub embedding_model: Option<String>,
}
#[derive(Debug, Default, Clone, Deserialize)]
pub struct EmbedResponse {
#[serde(default)]
pub embeddings: Vec<Vec<f64>>,
#[serde(default)]
pub embedding: Option<Vec<f64>>,
}
#[derive(Debug, Clone, Serialize)]
#[serde(untagged)]
pub enum Rerank {
Enabled(bool),
Config(RerankConfig),
}
#[derive(Debug, Default, Clone, Serialize)]
pub struct RerankConfig {
pub enabled: bool,
#[serde(skip_serializing_if = "Option::is_none")]
pub provider: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub model: Option<String>,
}
#[derive(Debug, Default, Clone, Serialize)]
pub struct SearchRequest {
#[serde(skip_serializing_if = "Option::is_none")]
pub query: Option<Vec<f64>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub query_text: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub embedding_model: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub embedding_provider: Option<String>,
pub top_k: u32,
#[serde(skip_serializing_if = "Option::is_none")]
pub filters: Option<HashMap<String, String>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub advanced_filters: Option<Vec<AdvancedFilter>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub nprobe_override: Option<u32>,
#[serde(skip_serializing_if = "Option::is_none")]
pub rerank_depth_override: Option<u32>,
#[serde(skip_serializing_if = "Option::is_none")]
pub hybrid: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub sparse_query: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub alpha: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub include_embeddings: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub include_documents: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub include_metadata: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub rerank: Option<Rerank>,
}
#[derive(Debug, Default, Clone, Serialize, Deserialize)]
pub struct AdvancedFilter {
pub field: String,
pub op: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub value: Option<Value>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub values: Option<Vec<Value>>,
}
#[derive(Debug, Default, Clone, Deserialize)]
pub struct SearchResult {
pub id: Value,
#[serde(default)]
pub score: f64,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub metadata: Option<Metadata>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub embedding: Option<Vec<f64>>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub doc_kind: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub doc_value: Option<String>,
}
#[derive(Debug, Default, Clone, Deserialize)]
pub struct SearchResponse {
#[serde(default)]
pub results: Vec<SearchResult>,
#[serde(default)]
pub dataset_id: Option<String>,
#[serde(default)]
pub query_time_ms: Option<f64>,
}
#[derive(Debug, Default, Clone, Deserialize)]
pub struct Source {
#[serde(default)]
pub id: Option<String>,
#[serde(default)]
pub source_id: Option<String>,
#[serde(default)]
pub uuid: Option<String>,
#[serde(default)]
pub name: String,
#[serde(default, rename = "type")]
pub kind: Option<String>,
#[serde(default)]
pub source_type: Option<String>,
#[serde(default)]
pub description: Option<String>,
#[serde(default)]
pub config: Option<HashMap<String, Value>>,
#[serde(default)]
pub metadata: Option<Metadata>,
#[serde(default)]
pub created_at: Option<String>,
#[serde(default)]
pub updated_at: Option<String>,
}
impl Source {
pub fn identifier(&self) -> Option<&str> {
self.id
.as_deref()
.or(self.source_id.as_deref())
.or(self.uuid.as_deref())
}
}
#[derive(Debug, Default, Clone, Deserialize)]
pub struct SourceList {
#[serde(default)]
pub sources: Vec<Source>,
#[serde(default)]
pub total: u32,
#[serde(default)]
pub limit: u32,
#[serde(default)]
pub offset: u32,
}
#[derive(Debug, Default, Clone, Serialize)]
pub struct CreateSourceRequest {
pub source_type: String,
pub name: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub description: Option<String>,
pub config: HashMap<String, Value>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub metadata: Option<Metadata>,
}
#[derive(Debug, Default, Clone, Deserialize)]
pub struct Job {
#[serde(default)]
pub job_id: String,
#[serde(default)]
pub status: String,
#[serde(default)]
pub message: Option<String>,
#[serde(default)]
pub documents_processed: Option<u64>,
#[serde(default)]
pub vectors_inserted: Option<u64>,
#[serde(default)]
pub processing_time_seconds: Option<f64>,
#[serde(default)]
pub pipeline_result: Option<Value>,
#[serde(default)]
pub error_details: Option<Value>,
#[serde(default)]
pub started_at: Option<String>,
#[serde(default)]
pub completed_at: Option<String>,
#[serde(default)]
pub progress_percentage: Option<f64>,
#[serde(default)]
pub current_step: Option<String>,
}
#[derive(Debug, Default, Clone, Deserialize)]
pub struct JobList {
#[serde(default)]
pub jobs: Vec<Job>,
#[serde(default)]
pub total: u32,
#[serde(default)]
pub limit: u32,
#[serde(default)]
pub offset: u32,
}
#[derive(Debug, Default, Clone, Serialize)]
pub struct StartIngestionRequest {
pub source_id: String,
pub dataset_id: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub pipeline_id: Option<String>,
}
#[derive(Debug, Default, Clone, Serialize)]
pub struct UploadFile {
pub name: String,
pub size_bytes: u64,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub content_type: Option<String>,
}
#[derive(Debug, Default, Clone, Serialize)]
pub(crate) struct InitUploadRequest {
pub files: Vec<UploadFile>,
}
#[derive(Debug, Default, Clone, Deserialize)]
pub struct UploadTarget {
pub file_id: String,
#[serde(default)]
pub file_name: Option<String>,
pub upload_url: String,
}
#[derive(Debug, Default, Clone, Deserialize)]
pub struct InitUploadResponse {
pub job_id: String,
#[serde(default)]
pub uploads: Vec<UploadTarget>,
}
#[derive(Debug, Default, Clone, Serialize)]
pub(crate) struct CompleteUploadRequest {
pub job_id: String,
pub file_ids: Vec<String>,
}
#[derive(Debug, Default, Clone, Serialize, Deserialize)]
pub struct ConversationMessage {
pub role: String,
pub content: String,
}
#[derive(Debug, Default, Clone, Serialize)]
pub struct AskRequest {
pub query: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub dataset_id: Option<Value>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub top_k: Option<u32>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub conversation_history: Option<Vec<ConversationMessage>>,
pub stream: bool,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub include_sources: Option<bool>,
}
#[derive(Debug, Default, Clone, Deserialize)]
pub struct AskResponse {
#[serde(default)]
pub answer: String,
#[serde(default)]
pub sources: Vec<SourceCitation>,
#[serde(default)]
pub chunks: Vec<RagChunk>,
#[serde(default)]
pub message: Option<String>,
#[serde(default)]
pub metadata: Option<Metadata>,
}
#[derive(Debug, Default, Clone, Deserialize)]
pub struct SourceCitation {
#[serde(default)]
pub name: Option<String>,
#[serde(default)]
pub path: Option<String>,
#[serde(default)]
pub url: Option<String>,
#[serde(default)]
pub source_type: Option<String>,
#[serde(default)]
pub content_type: Option<String>,
#[serde(default)]
pub relevance: Option<f64>,
#[serde(default)]
pub pages: Vec<u32>,
#[serde(default)]
pub chunk_count: Option<u32>,
#[serde(default)]
pub preview: Option<String>,
#[serde(default)]
pub chunks: Vec<RagChunk>,
#[serde(default)]
pub file_id: Option<String>,
#[serde(default)]
pub thumbnail_url: Option<String>,
}
#[derive(Debug, Default, Clone, Deserialize)]
pub struct RagChunk {
#[serde(default)]
pub id: Option<String>,
#[serde(default)]
pub chunk_id: Option<String>,
#[serde(default)]
pub text: Option<String>,
#[serde(default)]
pub score: Option<f64>,
#[serde(default)]
pub source: Option<String>,
#[serde(default)]
pub source_url: Option<String>,
#[serde(default)]
pub page: Option<Value>,
#[serde(default)]
pub metadata: Option<Metadata>,
}
#[derive(Debug, Default, Clone, Serialize)]
pub struct CreateSessionRequest {
#[serde(skip_serializing_if = "Option::is_none")]
pub title: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub workspace_id: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub dataset_id: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub metadata: Option<Metadata>,
}
impl From<()> for CreateSessionRequest {
fn from(_: ()) -> Self {
CreateSessionRequest::default()
}
}
impl From<&str> for CreateSessionRequest {
fn from(title: &str) -> Self {
CreateSessionRequest {
title: Some(title.to_owned()),
..Default::default()
}
}
}
impl From<String> for CreateSessionRequest {
fn from(title: String) -> Self {
CreateSessionRequest {
title: Some(title),
..Default::default()
}
}
}
#[derive(Debug, Default, Clone, Deserialize)]
pub struct IntelligenceSession {
#[serde(default)]
pub id: String,
#[serde(default)]
pub organization_id: Option<String>,
#[serde(default)]
pub user_id: Option<String>,
#[serde(default)]
pub workspace_id: Option<String>,
#[serde(default)]
pub dataset_id: Option<String>,
#[serde(default)]
pub title: Option<String>,
#[serde(default)]
pub status: Option<String>,
#[serde(default)]
pub metadata: Option<Metadata>,
#[serde(default)]
pub created_at: Option<String>,
#[serde(default)]
pub updated_at: Option<String>,
}
#[derive(Debug, Default, Clone, Deserialize)]
pub struct SessionList {
#[serde(default)]
pub sessions: Vec<IntelligenceSession>,
}
#[derive(Debug, Default, Clone, Serialize)]
pub struct AppendMessageRequest {
pub role: String,
pub content: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub metadata: Option<Metadata>,
}
#[derive(Debug, Default, Clone, Deserialize)]
pub struct SessionMessage {
#[serde(default)]
pub id: String,
#[serde(default)]
pub session_id: Option<String>,
#[serde(default)]
pub role: String,
#[serde(default)]
pub content: String,
#[serde(default)]
pub metadata: Option<Metadata>,
#[serde(default)]
pub created_at: Option<String>,
}
#[derive(Debug, Default, Clone, Deserialize)]
pub struct MessageList {
#[serde(default)]
pub messages: Vec<SessionMessage>,
}
#[derive(Debug, Default, Clone)]
pub struct DocumentListOpts {
pub limit: Option<u32>,
pub cursor: Option<String>,
pub status: Option<String>,
}
#[derive(Debug, Default, Clone, Deserialize)]
pub struct DatasetDocument {
pub id: String,
#[serde(default)]
pub dataset_id: Option<String>,
#[serde(default)]
pub source_id: Option<String>,
#[serde(default)]
pub source_type: Option<String>,
#[serde(default)]
pub external_id: Option<String>,
#[serde(default)]
pub file_name: Option<String>,
#[serde(default)]
pub mime_type: Option<String>,
#[serde(default)]
pub size_bytes: Option<i64>,
#[serde(default)]
pub content_hash: Option<String>,
#[serde(default)]
pub status: Option<String>,
#[serde(default)]
pub version: Option<i32>,
#[serde(default)]
pub chunk_count: Option<i32>,
#[serde(default)]
pub embeddings_count: Option<i32>,
#[serde(default)]
pub download_available: bool,
#[serde(default)]
pub created_at: Option<String>,
#[serde(default)]
pub updated_at: Option<String>,
}
#[derive(Debug, Default, Clone, Deserialize)]
pub struct Schedule {
#[serde(default)]
pub id: String,
#[serde(default)]
pub organization_id: Option<String>,
#[serde(default)]
pub source_id: Option<String>,
#[serde(default)]
pub dataset_id: Option<String>,
#[serde(default)]
pub pipeline_id: Option<String>,
#[serde(default)]
pub cron: Option<String>,
#[serde(default)]
pub timezone: Option<String>,
#[serde(default)]
pub enabled: bool,
#[serde(default)]
pub name: Option<String>,
#[serde(default)]
pub next_run_at: Option<String>,
#[serde(default)]
pub last_run_at: Option<String>,
#[serde(default)]
pub metadata: Option<Metadata>,
#[serde(default)]
pub created_at: Option<String>,
#[serde(default)]
pub updated_at: Option<String>,
}
#[derive(Debug, Default, Clone, Deserialize)]
pub struct ScheduleList {
#[serde(default)]
pub schedules: Vec<Schedule>,
#[serde(default)]
pub total: u32,
#[serde(default)]
pub limit: u32,
#[serde(default)]
pub offset: u32,
}
#[derive(Debug, Default, Clone, Serialize)]
pub struct CreateScheduleRequest {
pub source_id: String,
pub dataset_id: String,
pub cron: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub timezone: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub pipeline_id: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub enabled: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub name: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub metadata: Option<Metadata>,
}
#[derive(Debug, Default, Clone, Serialize)]
pub struct UpdateScheduleRequest {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub cron: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub timezone: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub pipeline_id: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub enabled: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub name: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub metadata: Option<Metadata>,
}
#[derive(Debug, Default, Clone, Deserialize)]
pub struct TriggerScheduleResponse {
#[serde(default)]
pub job_id: Option<String>,
}
#[derive(Debug, Default, Clone, Deserialize)]
pub struct DatasetDocumentList {
#[serde(default)]
pub documents: Vec<DatasetDocument>,
#[serde(default)]
pub next_cursor: Option<String>,
#[serde(default)]
pub limit: Option<u32>,
}