qdrant-edge 0.8.0

A lightweight, in-process vector search engine designed for embedded devices, autonomous systems, and mobile agents.
Documentation
use crate::common::universal_io::MmapFs;
use rand::distr::{Distribution, Uniform};
use rand::{Rng, RngExt};
use serde::{Deserialize, Serialize};
use serde_json::Map;
use tempfile::{Builder, TempDir};

use crate::blobstore::config::{
    Compression, DEFAULT_BLOCK_SIZE_BYTES, DEFAULT_REGION_SIZE_BLOCKS, GridstoreConfig,
    LogstoreConfig, Mode, StorageConfig,
};
use crate::blobstore::{Blob, Blobstore};

#[derive(Clone, Debug, Eq, PartialEq, Deserialize, Serialize)]
pub struct Payload(pub Map<String, serde_json::Value>);

impl Default for Payload {
    fn default() -> Self {
        Payload(serde_json::Map::new())
    }
}

impl Blob for Payload {
    fn to_bytes(&self) -> Vec<u8> {
        serde_json::to_vec(self).unwrap()
    }

    fn from_bytes(data: &[u8]) -> Self {
        serde_json::from_slice(data).unwrap()
    }
}

/// Create an empty storage with the default configuration
pub fn empty_storage() -> (TempDir, Blobstore<Payload>) {
    empty_storage_mode(Mode::Mutable)
}

/// Create an empty storage in append-only mode with the default configuration
pub fn empty_storage_append_only() -> (TempDir, Blobstore<Payload>) {
    empty_storage_mode(Mode::AppendOnly)
}

/// Create an empty storage in the given mode with the default configuration
pub fn empty_storage_mode(mode: Mode) -> (TempDir, Blobstore<Payload>) {
    let dir = Builder::new().prefix("test-storage").tempdir().unwrap();
    let storage = Blobstore::new(MmapFs, dir.path().to_path_buf(), default_config(mode)).unwrap();
    (dir, storage)
}

/// Default creation config for the given mode
pub fn default_config(mode: Mode) -> StorageConfig {
    match mode {
        Mode::Mutable => StorageConfig::Mutable(GridstoreConfig::DEFAULT),
        Mode::AppendOnly => StorageConfig::AppendOnly(LogstoreConfig::DEFAULT),
    }
}

/// Create an empty mutable storage with a specific page size
pub fn empty_storage_sized(
    page_size: usize,
    compression: Compression,
) -> (TempDir, Blobstore<Payload>) {
    let dir = Builder::new().prefix("test-storage").tempdir().unwrap();
    let config = StorageConfig::Mutable(GridstoreConfig {
        page_size_bytes: page_size,
        block_size_bytes: DEFAULT_BLOCK_SIZE_BYTES,
        region_size_blocks: DEFAULT_REGION_SIZE_BLOCKS,
        compression,
    });
    let storage = Blobstore::new(MmapFs, dir.path().to_path_buf(), config).unwrap();
    (dir, storage)
}

pub fn random_word(rng: &mut impl Rng) -> String {
    let len = rng.random_range(1..10);
    let mut word = String::with_capacity(len);
    for _ in 0..len {
        word.push(rng.random_range(b'a'..=b'z') as char);
    }
    word
}

pub fn random_payload(rng: &mut impl Rng, size_factor: usize) -> Payload {
    let mut payload = Payload::default();

    let word = random_word(rng);

    let sentence = (0..rng.random_range(1..20 * size_factor))
        .map(|_| random_word(rng))
        .collect::<Vec<_>>()
        .join(" ");

    let distr = Uniform::new(0, 100000).unwrap();
    let indices = (0..rng.random_range(1..100 * size_factor))
        .map(|_| distr.sample(rng))
        .collect::<Vec<_>>();

    payload.0 = serde_json::json!(
        {
            "word": word, // string
            "sentence": sentence, // string
            "number": rng.random_range(0..1000), // number
            "indices": indices, // array of numbers
            "bool": rng.random_bool(0.5), // boolean
            "null": serde_json::Value::Null, // null
            "object": {
                "bool": rng.random_bool(0.5),
            }, // object
        }
    )
    .as_object()
    .unwrap()
    .clone();

    payload
}

pub fn minimal_payload() -> Payload {
    Payload(serde_json::json!({"a": 1}).as_object().unwrap().clone())
}

pub const HM_FIELDS: [&str; 23] = [
    "article_id",
    "product_code",
    "prod_name",
    "product_type_no",
    "product_type_name",
    "product_group_name",
    "graphical_appearance_no",
    "graphical_appearance_name",
    "colour_group_code",
    "colour_group_name",
    "perceived_colour_value_id",
    "perceived_colour_value_name",
    "perceived_colour_master_id",
    "perceived_colour_master_name",
    "department_no",
    "department_name",
    "index_code,index_name",
    "index_group_no",
    "index_group_name",
    "section_no,section_name",
    "garment_group_no",
    "garment_group_name",
    "detail_desc",
];

#[cfg(test)]
mod tests {
    use crate::blobstore::Blob;
    use crate::blobstore::fixtures::Payload;

    #[test]
    fn test_serde_symmetry() {
        let mut payload = Payload::default();
        payload.0.insert(
            "key".to_string(),
            serde_json::Value::String("value".to_string()),
        );
        let bytes = payload.to_bytes();

        let deserialized = Payload::from_bytes(&bytes);
        assert_eq!(payload, deserialized);
    }
}