qdrant-edge 0.8.0

A lightweight, in-process vector search engine designed for embedded devices, autonomous systems, and mobile agents.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104

use std::io::Cursor;
use std::path::Path;

use crate::common::bitvec::BitVec;
use crate::common::generic_consts::Sequential;
use crate::common::mmap::AdviceSetting;
use crate::common::stored_bitslice::StoredBitSlice;
use crate::common::universal_io::{
    CachedReadFs, OpenOptions, Populate, ReadRange, TypedStorage, UniversalRead,
    UniversalReadFileOps, UniversalReadFs,
};

use super::ReadOnlyImmutableIdTracker;
use crate::segment::common::operation_error::OperationResult;
use crate::segment::id_tracker::compressed::versions_store::CompressedVersions;
use crate::segment::id_tracker::immutable_id_tracker::deleted_storage::deleted_path;
use crate::segment::id_tracker::immutable_id_tracker::mappings_storage::{load_mapping, mappings_path};
use crate::segment::id_tracker::immutable_id_tracker::versions_storage::version_mapping_path;
use crate::segment::types::SeqNumberType;

impl<S: UniversalRead> ReadOnlyImmutableIdTracker<S> {
    pub(super) fn open_options() -> OpenOptions {
        OpenOptions {
            writeable: false,
            need_sequential: false,
            populate: Populate::PreferBackground,
            advice: AdviceSetting::Global,
        }
    }

    /// Schedule background prefetch of every file [`open`](Self::open) will
    /// read
    ///
    /// Returns `false` (nothing scheduled) when the segment is not in the
    /// immutable format.
    pub fn try_preopen(
        fs: &impl CachedReadFs<File = S>,
        segment_path: &Path,
    ) -> OperationResult<bool> {
        if !UniversalReadFileOps::exists(fs, &mappings_path(segment_path))? {
            return Ok(false);
        }

        let options = Self::open_options();

        fs.schedule_prefetch(&deleted_path(segment_path), Some(options), None)?;
        fs.schedule_prefetch(&version_mapping_path(segment_path), Some(options), None)?;
        fs.schedule_prefetch(&mappings_path(segment_path), Some(options), None)?;

        Ok(true)
    }

    /// Open a read-only view over immutable ID tracker data at `segment_path`, threading every file
    /// open through `fs`. Read-only mirror of [`ImmutableIdTracker::open`]; it never writes.
    ///
    /// The `deleted` bitslice handle is kept for [`live_reload`](Self::live_reload); versions and
    /// mappings are immutable and read once into memory.
    ///
    /// [`ImmutableIdTracker::open`]: crate::id_tracker::immutable_id_tracker::ImmutableIdTracker::open
    ///
    /// Returns `Ok(None)` when the defining `id_tracker.mappings` file is absent
    /// (i.e. the segment is not in the immutable format). The probe uses
    /// `exists`, which a caching `fs` answers from its listing snapshot — a
    /// probe-by-open would consume the file's prefetched take-once handle
    /// that [`Self::open`] needs right after.
    pub fn try_open(
        fs: &impl UniversalReadFs<File = S>,
        segment_path: &Path,
    ) -> OperationResult<Option<Self>> {
        if !UniversalReadFileOps::exists(fs, &mappings_path(segment_path))? {
            return Ok(None);
        }
        Ok(Some(Self::open(fs, segment_path)?))
    }

    pub fn open(fs: &impl UniversalReadFs<File = S>, segment_path: &Path) -> OperationResult<Self> {
        let options = Self::open_options();

        let deleted =
            StoredBitSlice::open(fs, deleted_path(segment_path), options, Default::default())?;
        let mut deleted_bitvec = BitVec::new();
        deleted_bitvec.extend_from_bitslice(deleted.read_all()?.as_ref());

        let internal_to_version_file = TypedStorage::<S, SeqNumberType>::new(fs.open(
            version_mapping_path(segment_path),
            options,
            Default::default(),
        )?);
        let internal_to_version =
            CompressedVersions::from_slice(&internal_to_version_file.read_whole()?);

        let mappings_file = fs.open(mappings_path(segment_path), options, Default::default())?;
        let mappings_bytes = mappings_file
            .read::<_, u8>(ReadRange::new(0, mappings_file.len::<u8>()?), Sequential)?;
        let mappings = load_mapping(Cursor::new(mappings_bytes.as_ref()), Some(deleted_bitvec))?;

        Ok(Self {
            path: segment_path.to_path_buf(),
            deleted,
            internal_to_version,
            mappings,
        })
    }
}