lvv 0.3.0

A pipeline for embedding datasets with LLMs (Ollama/OpenAI) and loading them into a Qdrant vector database, with caching and job queuing.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32

// TODO: reemplazar anyhow con thiserror
use anyhow::Context;
use serde::{Deserialize, Serialize};
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DataSet<T> {
    pub filename: String,
    pub identifier: String,
    pub data: Option<Vec<T>>,
}

impl<T> DataSet<T>
where
    T: Serialize + Clone,
{
    pub fn new(filename: impl Into<String>, identifier: impl Into<String>, data: Vec<T>) -> Self {
        let data = Some(data);
        Self {
            filename: filename.into(),
            identifier: identifier.into(),
            data,
        }
    }
    pub fn serialize_to_vec(self) -> anyhow::Result<Vec<String>> {
        let mut vec = vec![];
        let data = self.data.context("No data")?;
        for datum in data {
            let string = serde_json::to_string(&datum).context("Couldn't serialize a node")?;
            vec.push(string);
        }
        Ok(vec)
    }
}