lvv 0.3.0

A pipeline for embedding datasets with LLMs (Ollama/OpenAI) and loading them into a Qdrant vector database, with caching and job queuing.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31

use std::{fs::OpenOptions, io::Write};

use clap::Parser;
use lvv::cache::cache_embeddings::Cache;
fn main() {
    let eve = LvvExtendCache::parse();
    println!("{eve:?}");
    let filename = eve.filename.unwrap_or("joined_cache.json".to_owned());
    let mut file_base =
        Cache::from_json_file(eve.base.as_str()).expect("No se encontro archivo base");
    eve.others
        .into_iter()
        .map(|file| Cache::from_json_file(&file).unwrap())
        .for_each(|cache| file_base.cache.extend(cache.cache));
    let mut file = OpenOptions::new()
        .create(true)
        .truncate(true)
        .write(true)
        .open(filename)
        .unwrap();

    let st = serde_json::to_string(&file_base).unwrap();
    file.write_all(st.as_bytes()).unwrap();
}

#[derive(Debug, Parser)]
pub struct LvvExtendCache {
    pub base: String,
    pub others: Vec<String>,
    pub filename: Option<String>,
}