lvv/points/mod.rs
1//! Turn typed records into vector points.
2//!
3//! A [`VectorDatabaseItem`] is a record that becomes one point: a category, a
4//! text description to embed, and a Qdrant [`Payload`] to store next to the
5//! vector. A [`VectorDatabase`] holds items and lists all of their points at
6//! once. Both produce [`VectorPointDraft`]s, which you embed with
7//! [`EmbeddingProvider`](crate::inference::EmbeddingProvider) and write with a
8//! [`Sink`](crate::db::Sink).
9//!
10//! # Deriving
11//!
12//! The `derive` feature re-exports the `VectorDatabaseItem` and
13//! `VectorDatabase` derive macros from
14//! [`lvv-macros`](https://docs.rs/lvv-macros) in this module, so one import
15//! brings in both a trait and its derive:
16//!
17//! ```toml
18//! [dependencies]
19//! lvv = { version = "0.5", features = ["derive"] }
20//! ```
21//!
22//! Mark the fields to embed with `#[lvv(description)]`, hide fields with
23//! `#[lvv(skip)]`, and change payload keys with `#[lvv(rename = "...")]`:
24//!
25#![cfg_attr(feature = "derive", doc = "```")]
26#![cfg_attr(not(feature = "derive"), doc = "```ignore")]
27//! use lvv::points::{VectorDatabase, VectorDatabaseItem};
28//! use serde::{Deserialize, Serialize};
29//!
30//! #[derive(Serialize, Deserialize, VectorDatabaseItem)]
31//! #[serde(rename_all = "camelCase")]
32//! struct Position {
33//! // Embedded, and stored under `role` instead of `jobTitle`.
34//! #[lvv(description, rename = "role")]
35//! job_title: String,
36//! // Embedded when present; `None` adds nothing to the description.
37//! #[lvv(description)]
38//! organization: Option<String>,
39//! // Stored as `startedOn`, following serde.
40//! started_on: String,
41//! // Neither embedded nor stored.
42//! #[lvv(skip)]
43//! internal_id: u64,
44//! }
45//!
46//! #[derive(Serialize, Deserialize, VectorDatabaseItem)]
47//! struct Interest(#[lvv(description)] String);
48//!
49//! #[derive(VectorDatabase)]
50//! struct Portfolio {
51//! positions: Vec<Position>,
52//! interests: Option<Vec<Interest>>,
53//! #[lvv(skip)]
54//! owner: String,
55//! }
56//!
57//! # fn main() -> anyhow::Result<()> {
58//! let portfolio = Portfolio {
59//! positions: vec![Position {
60//! job_title: "Researcher".into(),
61//! organization: Some("UNLP".into()),
62//! started_on: "2020-03-01".into(),
63//! internal_id: 42,
64//! }],
65//! interests: Some(vec![Interest("Vector search".into())]),
66//! owner: "Ada".into(),
67//! };
68//!
69//! let drafts = portfolio.point_drafts()?;
70//! assert_eq!(drafts.len(), 2);
71//!
72//! let position = &drafts[0];
73//! assert_eq!(position.category, "Position");
74//! assert_eq!(position.description, "Researcher\nUNLP");
75//! assert_eq!(
76//! serde_json::to_value(&position.payload)?,
77//! serde_json::json!({
78//! "role": "Researcher",
79//! "organization": "UNLP",
80//! "startedOn": "2020-03-01",
81//! "category": "Position",
82//! }),
83//! );
84//! assert_eq!(drafts[1].category, "Interest");
85//! # Ok(())
86//! # }
87//! ```
88//!
89//! See the [`lvv-macros` documentation](https://docs.rs/lvv-macros) for every
90//! attribute and the field types `VectorDatabase` accepts, and
91//! [`examples/derive.rs`](https://github.com/egonik-unlp/lvv/blob/main/examples/derive.rs)
92//! for a complete pipeline that loads derived records from files, embeds them
93//! and runs a `JobQueue` into Qdrant.
94//!
95//! # Implementing by hand
96//!
97//! ```
98//! use lvv::points::VectorDatabaseItem;
99//! use serde::{Deserialize, Serialize};
100//!
101//! #[derive(Serialize, Deserialize)]
102//! struct Skill {
103//! name: String,
104//! level: String,
105//! }
106//!
107//! impl VectorDatabaseItem for Skill {
108//! fn category(&self) -> String {
109//! "skill".into()
110//! }
111//!
112//! fn into_description(&self) -> String {
113//! format!("{} ({})", self.name, self.level)
114//! }
115//! }
116//!
117//! # fn main() -> anyhow::Result<()> {
118//! let skill = Skill { name: "Rust".into(), level: "advanced".into() };
119//! let draft = skill.try_into_database_item()?;
120//! assert_eq!(draft.description, "Rust (advanced)");
121//! # Ok(())
122//! # }
123//! ```
124//!
125//! # Embedding and storing drafts
126//!
127//! Embed the descriptions, then write the payloads as rows in the same order:
128//!
129//! ```no_run
130//! use lvv::{
131//! db::{Distance, QdrantSink, Sink, SinkContext},
132//! db::vector_database::{DatabaseParams, Location},
133//! inference::EmbeddingProvider,
134//! points::VectorPointDraft,
135//! };
136//!
137//! # async fn store(drafts: Vec<VectorPointDraft>) -> anyhow::Result<()> {
138//! let descriptions: Vec<&str> = drafts.iter().map(|d| d.description.as_str()).collect();
139//! let embeddings = EmbeddingProvider::new("nomic-embed-text")
140//! .embed_texts(&descriptions)
141//! .await?;
142//! let rows = drafts
143//! .iter()
144//! .map(|d| serde_json::to_value(&d.payload))
145//! .collect::<Result<Vec<_>, _>>()?;
146//!
147//! let sink = QdrantSink::new(DatabaseParams::new(
148//! Location::new_local("http://localhost:6334"),
149//! "portfolio".into(),
150//! Distance::Cosine,
151//! 768,
152//! ));
153//! sink.write(&SinkContext {
154//! collection_name: "portfolio",
155//! dims: 768,
156//! extends: true,
157//! embeddings: &embeddings,
158//! rows: &rows,
159//! })
160//! .await?;
161//! # Ok(())
162//! # }
163//! ```
164
165use qdrant_client::{Payload, QdrantError};
166use serde::{Serialize, de::DeserializeOwned};
167
168#[cfg(feature = "derive")]
169pub use lvv_macros::{VectorDatabase, VectorDatabaseItem};
170
171/// Why a record couldn't become a [`VectorPointDraft`].
172#[derive(Debug, thiserror::Error)]
173pub enum PointError {
174 /// The record couldn't be serialized as JSON.
175 #[error("could not serialize the record: {0}")]
176 Serialize(#[from] serde_json::Error),
177 /// The record serialized to a JSON value of an unexpected shape, such as
178 /// a string where an object was needed.
179 #[error("`{category}` did not serialize to a JSON {expected}")]
180 Shape {
181 /// The record's category.
182 category: String,
183 /// The JSON shape the payload needs, such as `object`.
184 expected: String,
185 },
186 /// Qdrant rejected the payload.
187 #[error("invalid Qdrant payload: {0}")]
188 Payload(Box<QdrantError>),
189}
190
191impl From<QdrantError> for PointError {
192 fn from(err: QdrantError) -> Self {
193 PointError::Payload(Box::new(err))
194 }
195}
196
197/// A record that becomes one vector point.
198///
199/// Implement [`category`](Self::category) and
200/// [`into_description`](Self::into_description); the payload defaults to the
201/// record's JSON serialization. With the `derive` feature,
202/// `#[derive(VectorDatabaseItem)]` generates all three from field attributes.
203/// See the [module documentation](crate::points).
204pub trait VectorDatabaseItem: DeserializeOwned + Serialize {
205 /// The kind of record, used to group or filter points. The derive uses the
206 /// struct name.
207 fn category(&self) -> String;
208 /// The text to embed for this record.
209 fn into_description(&self) -> String;
210 /// The data stored next to the vector.
211 ///
212 /// Defaults to the record serialized with `serde_json`.
213 ///
214 /// # Errors
215 ///
216 /// Returns an error if the record doesn't serialize to a JSON object.
217 fn into_payload(&self) -> Result<Payload, PointError> {
218 Ok(Payload::try_from(serde_json::to_value(self)?)?)
219 }
220 /// Builds this record's [`VectorPointDraft`] from the methods above.
221 ///
222 /// # Errors
223 ///
224 /// Returns the error from [`into_payload`](Self::into_payload).
225 fn try_into_database_item(&self) -> Result<VectorPointDraft, PointError> {
226 let payload = self.into_payload()?;
227 Ok(VectorPointDraft {
228 category: self.category(),
229 description: self.into_description(),
230 payload,
231 })
232 }
233}
234
235/// Renders a field as text for a derived [`VectorDatabaseItem`] description.
236///
237/// Every field marked `#[lvv(description)]` must implement this trait. It is
238/// implemented for `String`, integer and float primitives, `Option<T>` (empty
239/// for `None`) and `Vec<T>` (one line per non-empty element). Implement it for your own
240/// types:
241///
242/// ```
243/// use lvv::points::IntoDescriptionValue;
244/// use serde::Serialize;
245///
246/// #[derive(Serialize)]
247/// enum Level {
248/// Beginner,
249/// Expert,
250/// }
251///
252/// impl IntoDescriptionValue for Level {
253/// fn into_description_value(&self) -> String {
254/// match self {
255/// Level::Beginner => "beginner".into(),
256/// Level::Expert => "expert".into(),
257/// }
258/// }
259/// }
260///
261/// assert_eq!(Some(Level::Expert).into_description_value(), "expert");
262/// ```
263pub trait IntoDescriptionValue: Serialize {
264 /// The text for this value. An empty string leaves the field out of the
265 /// description.
266 fn into_description_value(&self) -> String;
267}
268
269impl IntoDescriptionValue for String {
270 fn into_description_value(&self) -> String {
271 self.to_string()
272 }
273}
274
275impl<T> IntoDescriptionValue for Option<T>
276where
277 T: IntoDescriptionValue,
278{
279 fn into_description_value(&self) -> String {
280 match self {
281 Some(inner_value) => inner_value.into_description_value(),
282 None => "".to_string(),
283 }
284 }
285}
286impl<T> IntoDescriptionValue for Vec<T>
287where
288 T: IntoDescriptionValue,
289{
290 fn into_description_value(&self) -> String {
291 self.iter()
292 .map(IntoDescriptionValue::into_description_value)
293 .filter(|value| !value.is_empty())
294 .collect::<Vec<_>>()
295 .join("\n")
296 }
297}
298
299macro_rules! impl_into_description_value_for_num {
300 ($($t:ty),* $(,)?) => {
301 $(
302 impl IntoDescriptionValue for $t {
303 fn into_description_value(&self) -> String {
304 format!("{:?}", self)
305 }
306 }
307 )*
308 };
309}
310
311impl_into_description_value_for_num!(
312 u8, u16, u32, u64, u128, usize, i8, i16, i32, i64, i128, isize, f32, f64,
313);
314/// A container of [`VectorDatabaseItem`]s that lists all of their points.
315///
316/// With the `derive` feature, `#[derive(VectorDatabase)]` implements this for a
317/// struct whose fields hold items directly or in a `Vec`, `BTreeMap`, `HashMap`
318/// or `Option`.
319pub trait VectorDatabase {
320 /// One draft per item, in field order.
321 ///
322 /// # Errors
323 ///
324 /// Returns an error if any item's payload can't be built.
325 fn point_drafts(&self) -> Result<Vec<VectorPointDraft>, PointError>;
326}
327
328/// A vector point before embedding: what to embed and what to store with it.
329#[derive(Debug)]
330pub struct VectorPointDraft {
331 /// The kind of record, from [`VectorDatabaseItem::category`].
332 pub category: String,
333 /// The text to embed.
334 pub description: String,
335 /// The data stored next to the vector.
336 pub payload: Payload,
337}
338
339impl VectorPointDraft {
340 /// Creates a draft from its parts.
341 pub fn new(category: String, description: String, payload: Payload) -> Self {
342 Self {
343 category,
344 description,
345 payload,
346 }
347 }
348}