Skip to main content

lvv/points/
mod.rs

1//! Turn typed records into vector points.
2//!
3//! A [`VectorDatabaseItem`] is a record that becomes one point: a category, a
4//! text description to embed, and a Qdrant [`Payload`] to store next to the
5//! vector. A [`VectorDatabase`] holds items and lists all of their points at
6//! once. Both produce [`VectorPointDraft`]s, which you embed with
7//! [`EmbeddingProvider`](crate::inference::EmbeddingProvider) and write with a
8//! [`Sink`](crate::db::Sink).
9//!
10//! # Deriving
11//!
12//! The `derive` feature re-exports the `VectorDatabaseItem` and
13//! `VectorDatabase` derive macros from
14//! [`lvv-macros`](https://docs.rs/lvv-macros) in this module, so one import
15//! brings in both a trait and its derive:
16//!
17//! ```toml
18//! [dependencies]
19//! lvv = { version = "0.5", features = ["derive"] }
20//! ```
21//!
22//! Mark the fields to embed with `#[lvv(description)]`, hide fields with
23//! `#[lvv(skip)]`, and change payload keys with `#[lvv(rename = "...")]`:
24//!
25#![cfg_attr(feature = "derive", doc = "```")]
26#![cfg_attr(not(feature = "derive"), doc = "```ignore")]
27//! use lvv::points::{VectorDatabase, VectorDatabaseItem};
28//! use serde::{Deserialize, Serialize};
29//!
30//! #[derive(Serialize, Deserialize, VectorDatabaseItem)]
31//! #[serde(rename_all = "camelCase")]
32//! struct Position {
33//!     // Embedded, and stored under `role` instead of `jobTitle`.
34//!     #[lvv(description, rename = "role")]
35//!     job_title: String,
36//!     // Embedded when present; `None` adds nothing to the description.
37//!     #[lvv(description)]
38//!     organization: Option<String>,
39//!     // Stored as `startedOn`, following serde.
40//!     started_on: String,
41//!     // Neither embedded nor stored.
42//!     #[lvv(skip)]
43//!     internal_id: u64,
44//! }
45//!
46//! #[derive(Serialize, Deserialize, VectorDatabaseItem)]
47//! struct Interest(#[lvv(description)] String);
48//!
49//! #[derive(VectorDatabase)]
50//! struct Portfolio {
51//!     positions: Vec<Position>,
52//!     interests: Option<Vec<Interest>>,
53//!     #[lvv(skip)]
54//!     owner: String,
55//! }
56//!
57//! # fn main() -> anyhow::Result<()> {
58//! let portfolio = Portfolio {
59//!     positions: vec![Position {
60//!         job_title: "Researcher".into(),
61//!         organization: Some("UNLP".into()),
62//!         started_on: "2020-03-01".into(),
63//!         internal_id: 42,
64//!     }],
65//!     interests: Some(vec![Interest("Vector search".into())]),
66//!     owner: "Ada".into(),
67//! };
68//!
69//! let drafts = portfolio.point_drafts()?;
70//! assert_eq!(drafts.len(), 2);
71//!
72//! let position = &drafts[0];
73//! assert_eq!(position.category, "Position");
74//! assert_eq!(position.description, "Researcher\nUNLP");
75//! assert_eq!(
76//!     serde_json::to_value(&position.payload)?,
77//!     serde_json::json!({
78//!         "role": "Researcher",
79//!         "organization": "UNLP",
80//!         "startedOn": "2020-03-01",
81//!         "category": "Position",
82//!     }),
83//! );
84//! assert_eq!(drafts[1].category, "Interest");
85//! # Ok(())
86//! # }
87//! ```
88//!
89//! See the [`lvv-macros` documentation](https://docs.rs/lvv-macros) for every
90//! attribute and the field types `VectorDatabase` accepts, and
91//! [`examples/derive.rs`](https://github.com/egonik-unlp/lvv/blob/main/examples/derive.rs)
92//! for a complete pipeline that loads derived records from files, embeds them
93//! and runs a `JobQueue` into Qdrant.
94//!
95//! # Implementing by hand
96//!
97//! ```
98//! use lvv::points::VectorDatabaseItem;
99//! use serde::{Deserialize, Serialize};
100//!
101//! #[derive(Serialize, Deserialize)]
102//! struct Skill {
103//!     name: String,
104//!     level: String,
105//! }
106//!
107//! impl VectorDatabaseItem for Skill {
108//!     fn category(&self) -> String {
109//!         "skill".into()
110//!     }
111//!
112//!     fn into_description(&self) -> String {
113//!         format!("{} ({})", self.name, self.level)
114//!     }
115//! }
116//!
117//! # fn main() -> anyhow::Result<()> {
118//! let skill = Skill { name: "Rust".into(), level: "advanced".into() };
119//! let draft = skill.try_into_database_item()?;
120//! assert_eq!(draft.description, "Rust (advanced)");
121//! # Ok(())
122//! # }
123//! ```
124//!
125//! # Embedding and storing drafts
126//!
127//! Embed the descriptions, then write the payloads as rows in the same order:
128//!
129//! ```no_run
130//! use lvv::{
131//!     db::{Distance, QdrantSink, Sink, SinkContext},
132//!     db::vector_database::{DatabaseParams, Location},
133//!     inference::EmbeddingProvider,
134//!     points::VectorPointDraft,
135//! };
136//!
137//! # async fn store(drafts: Vec<VectorPointDraft>) -> anyhow::Result<()> {
138//! let descriptions: Vec<&str> = drafts.iter().map(|d| d.description.as_str()).collect();
139//! let embeddings = EmbeddingProvider::new("nomic-embed-text")
140//!     .embed_texts(&descriptions)
141//!     .await?;
142//! let rows = drafts
143//!     .iter()
144//!     .map(|d| serde_json::to_value(&d.payload))
145//!     .collect::<Result<Vec<_>, _>>()?;
146//!
147//! let sink = QdrantSink::new(DatabaseParams::new(
148//!     Location::new_local("http://localhost:6334"),
149//!     "portfolio".into(),
150//!     Distance::Cosine,
151//!     768,
152//! ));
153//! sink.write(&SinkContext {
154//!     collection_name: "portfolio",
155//!     dims: 768,
156//!     extends: true,
157//!     embeddings: &embeddings,
158//!     rows: &rows,
159//! })
160//! .await?;
161//! # Ok(())
162//! # }
163//! ```
164
165use qdrant_client::{Payload, QdrantError};
166use serde::{Serialize, de::DeserializeOwned};
167
168#[cfg(feature = "derive")]
169pub use lvv_macros::{VectorDatabase, VectorDatabaseItem};
170
171/// Why a record couldn't become a [`VectorPointDraft`].
172#[derive(Debug, thiserror::Error)]
173pub enum PointError {
174    /// The record couldn't be serialized as JSON.
175    #[error("could not serialize the record: {0}")]
176    Serialize(#[from] serde_json::Error),
177    /// The record serialized to a JSON value of an unexpected shape, such as
178    /// a string where an object was needed.
179    #[error("`{category}` did not serialize to a JSON {expected}")]
180    Shape {
181        /// The record's category.
182        category: String,
183        /// The JSON shape the payload needs, such as `object`.
184        expected: String,
185    },
186    /// Qdrant rejected the payload.
187    #[error("invalid Qdrant payload: {0}")]
188    Payload(Box<QdrantError>),
189}
190
191impl From<QdrantError> for PointError {
192    fn from(err: QdrantError) -> Self {
193        PointError::Payload(Box::new(err))
194    }
195}
196
197/// A record that becomes one vector point.
198///
199/// Implement [`category`](Self::category) and
200/// [`into_description`](Self::into_description); the payload defaults to the
201/// record's JSON serialization. With the `derive` feature,
202/// `#[derive(VectorDatabaseItem)]` generates all three from field attributes.
203/// See the [module documentation](crate::points).
204pub trait VectorDatabaseItem: DeserializeOwned + Serialize {
205    /// The kind of record, used to group or filter points. The derive uses the
206    /// struct name.
207    fn category(&self) -> String;
208    /// The text to embed for this record.
209    fn into_description(&self) -> String;
210    /// The data stored next to the vector.
211    ///
212    /// Defaults to the record serialized with `serde_json`.
213    ///
214    /// # Errors
215    ///
216    /// Returns an error if the record doesn't serialize to a JSON object.
217    fn into_payload(&self) -> Result<Payload, PointError> {
218        Ok(Payload::try_from(serde_json::to_value(self)?)?)
219    }
220    /// Builds this record's [`VectorPointDraft`] from the methods above.
221    ///
222    /// # Errors
223    ///
224    /// Returns the error from [`into_payload`](Self::into_payload).
225    fn try_into_database_item(&self) -> Result<VectorPointDraft, PointError> {
226        let payload = self.into_payload()?;
227        Ok(VectorPointDraft {
228            category: self.category(),
229            description: self.into_description(),
230            payload,
231        })
232    }
233}
234
235/// Renders a field as text for a derived [`VectorDatabaseItem`] description.
236///
237/// Every field marked `#[lvv(description)]` must implement this trait. It is
238/// implemented for `String`, integer and float primitives, `Option<T>` (empty
239/// for `None`) and `Vec<T>` (one line per non-empty element). Implement it for your own
240/// types:
241///
242/// ```
243/// use lvv::points::IntoDescriptionValue;
244/// use serde::Serialize;
245///
246/// #[derive(Serialize)]
247/// enum Level {
248///     Beginner,
249///     Expert,
250/// }
251///
252/// impl IntoDescriptionValue for Level {
253///     fn into_description_value(&self) -> String {
254///         match self {
255///             Level::Beginner => "beginner".into(),
256///             Level::Expert => "expert".into(),
257///         }
258///     }
259/// }
260///
261/// assert_eq!(Some(Level::Expert).into_description_value(), "expert");
262/// ```
263pub trait IntoDescriptionValue: Serialize {
264    /// The text for this value. An empty string leaves the field out of the
265    /// description.
266    fn into_description_value(&self) -> String;
267}
268
269impl IntoDescriptionValue for String {
270    fn into_description_value(&self) -> String {
271        self.to_string()
272    }
273}
274
275impl<T> IntoDescriptionValue for Option<T>
276where
277    T: IntoDescriptionValue,
278{
279    fn into_description_value(&self) -> String {
280        match self {
281            Some(inner_value) => inner_value.into_description_value(),
282            None => "".to_string(),
283        }
284    }
285}
286impl<T> IntoDescriptionValue for Vec<T>
287where
288    T: IntoDescriptionValue,
289{
290    fn into_description_value(&self) -> String {
291        self.iter()
292            .map(IntoDescriptionValue::into_description_value)
293            .filter(|value| !value.is_empty())
294            .collect::<Vec<_>>()
295            .join("\n")
296    }
297}
298
299macro_rules! impl_into_description_value_for_num {
300    ($($t:ty),* $(,)?) => {
301        $(
302            impl IntoDescriptionValue for $t {
303                fn into_description_value(&self) -> String {
304                    format!("{:?}", self)
305                }
306            }
307        )*
308    };
309}
310
311impl_into_description_value_for_num!(
312    u8, u16, u32, u64, u128, usize, i8, i16, i32, i64, i128, isize, f32, f64,
313);
314/// A container of [`VectorDatabaseItem`]s that lists all of their points.
315///
316/// With the `derive` feature, `#[derive(VectorDatabase)]` implements this for a
317/// struct whose fields hold items directly or in a `Vec`, `BTreeMap`, `HashMap`
318/// or `Option`.
319pub trait VectorDatabase {
320    /// One draft per item, in field order.
321    ///
322    /// # Errors
323    ///
324    /// Returns an error if any item's payload can't be built.
325    fn point_drafts(&self) -> Result<Vec<VectorPointDraft>, PointError>;
326}
327
328/// A vector point before embedding: what to embed and what to store with it.
329#[derive(Debug)]
330pub struct VectorPointDraft {
331    /// The kind of record, from [`VectorDatabaseItem::category`].
332    pub category: String,
333    /// The text to embed.
334    pub description: String,
335    /// The data stored next to the vector.
336    pub payload: Payload,
337}
338
339impl VectorPointDraft {
340    /// Creates a draft from its parts.
341    pub fn new(category: String, description: String, payload: Payload) -> Self {
342        Self {
343            category,
344            description,
345            payload,
346        }
347    }
348}