seismic 0.2.1

Seismic is designed for effective and efficient KNN retrieval over learned sparse embeddings.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65

use std::{collections::HashMap, fs::File, io::BufReader};

use serde::Deserialize;

//use qwt::SpaceUsage as SpaceUsageQwt;
use serde_json::Deserializer;

use crate::DataType;

#[derive(Debug, Deserialize)]
#[serde(untagged)] // This allows deserializing without tagging the enum variants
pub enum DocIdType {
    StringId(String),
    UsizeId(usize),
}

//TODO: change this name with a more meaningful one
#[derive(Debug, Deserialize)]
pub struct JsonFormat {
    id: DocIdType,
    vector: HashMap<String, f32>,
}

impl JsonFormat {
    pub fn vector(&self) -> &HashMap<String, f32> {
        &self.vector
    }

    //TODO: this allows to read docids either as a string or as an integer, but we always store them as
    // strings in memory.
    pub fn get_id_as_string(&self) -> String {
        match &self.id {
            DocIdType::StringId(s) => s.clone(),    // Clone the String
            DocIdType::UsizeId(n) => n.to_string(), // Convert usize to String
        }
    }
}

pub fn extract_jsonl<T>(current_jsonl: JsonFormat) -> (String, Vec<String>, Vec<T>)
where
    T: DataType,
{
    let (coords, values): (Vec<_>, Vec<_>) = current_jsonl
        .vector()
        .iter()
        .map(|(s, y)| (s.to_string(), T::from_f32(*y).unwrap()))
        .unzip();

    (current_jsonl.get_id_as_string(), coords, values)
}

pub fn read_queries(input_file: &String) -> Vec<(String, Vec<String>, Vec<f32>)> {
    let f = File::open(input_file).expect(&format!("Unable to open {}", input_file));
    let reader = BufReader::new(f);
    let stream: serde_json::StreamDeserializer<
        serde_json::de::IoRead<BufReader<File>>,
        JsonFormat,
    > = Deserializer::from_reader(reader).into_iter();

    let values = stream
        .into_iter()
        .map(|x| (extract_jsonl(x.unwrap())))
        .collect();
    values
}