qdrant_edge/edge/read_only/
enumerate.rs

1use std::collections::HashMap;
2use std::path::{Path, PathBuf};
3
4use crate::common::universal_io::{UniversalReadFs, read_json_via};
5use crate::segment::common::operation_error::OperationResult;
6use crate::shard::files::{SEGMENTS_PATH, segment_manifest_path};
7use crate::shard::segment_manifest::SegmentsManifest;
8use uuid::Uuid;
9
10use crate::edge::edge_shard::scan_segment_dirs;
11
12/// Enumerates the segments that make up the shard, keyed by their UUID.
13///
14/// The follower delegates discovery to an enumerator, chosen by whoever knows the backend:
15///
16/// * the default ([`ManifestSegmentEnumerator`], wired by
17///   [`open_mmap`](super::ReadOnlyEdgeShard::open_mmap)) reads the leader's segment manifest;
18/// * [`LocalSegmentEnumerator`] scans the local `segments/` directory;
19/// * an S3 follower can supply its own (e.g. reading the manifest over object storage).
20///
21/// Called on every [`refresh`](super::ReadOnlyEdgeShard::refresh), so it must reflect the current
22/// set. The returned paths are segment directory paths interpreted relative to the backend root
23/// (e.g. `segments/<uuid>`), matching what [`ReadOnlySegment::open`] expects.
24///
25/// [`ReadOnlySegment::open`]: segment::segment::read_only::ReadOnlySegment::open
26pub trait SegmentEnumerator: Send + Sync {
27    fn list_segments(&self) -> OperationResult<HashMap<Uuid, PathBuf>>;
28}
29
30/// [`SegmentEnumerator`] that reads the leader's segment manifest (`segments_manifest.json`, sitting
31/// next to the `segments/` directory) and returns its readable segments — `active`, plus
32/// `optimizing` ones which stay live until their rebuild's swap. Errors if no manifest is present:
33/// the manifest is the source of truth, so a follower using this enumerator requires the leader to
34/// write one (the `write_segment_manifest` feature flag).
35///
36/// Generic over the read backend `F` (a [`UniversalReadFs`]), so it reads the manifest over any
37/// storage — local memory-mapped files ([`MmapFs`](common::universal_io::MmapFs)) or a blob/S3
38/// backend alike. Wired with `MmapFs` by
39/// [`ReadOnlyEdgeShard::open_mmap`](super::ReadOnlyEdgeShard::open_mmap); an object-storage follower
40/// constructs it with its own blob filesystem.
41pub struct ManifestSegmentEnumerator<F: UniversalReadFs> {
42    /// Read backend used to read the manifest file.
43    fs: F,
44    /// The segment manifest, sitting next to (not inside) the `segments/` directory.
45    manifest_path: PathBuf,
46    /// The `segments/` directory; segment directories live under it as `segments/<uuid>`.
47    segments_path: PathBuf,
48}
49
50impl<F: UniversalReadFs> ManifestSegmentEnumerator<F> {
51    /// `shard_path` is the shard root (the directory containing `segments/`); `fs` is the backend the
52    /// manifest is read through.
53    pub fn new(fs: F, shard_path: &Path) -> Self {
54        Self {
55            fs,
56            manifest_path: segment_manifest_path(shard_path),
57            segments_path: shard_path.join(SEGMENTS_PATH),
58        }
59    }
60}
61
62impl<F: UniversalReadFs + Send + Sync> SegmentEnumerator for ManifestSegmentEnumerator<F> {
63    fn list_segments(&self) -> OperationResult<HashMap<Uuid, PathBuf>> {
64        let manifest: SegmentsManifest = read_json_via(&self.fs, &self.manifest_path)?;
65        Ok(manifest
66            .iter()
67            .filter(|(_, state)| state.is_usable())
68            .map(|(uuid, _)| (*uuid, self.segments_path.join(uuid.to_string())))
69            .collect())
70    }
71}
72
73/// [`SegmentEnumerator`] for local filesystems: scans the `segments/` directory. Wired
74/// automatically by [`ReadOnlyEdgeShard::open_mmap`](super::ReadOnlyEdgeShard::open_mmap).
75pub struct LocalSegmentEnumerator {
76    segments_path: PathBuf,
77}
78
79impl LocalSegmentEnumerator {
80    /// `shard_path` is the shard root (the directory containing `segments/`).
81    pub fn new(shard_path: &std::path::Path) -> Self {
82        Self {
83            segments_path: shard_path.join(SEGMENTS_PATH),
84        }
85    }
86}
87
88impl SegmentEnumerator for LocalSegmentEnumerator {
89    fn list_segments(&self) -> OperationResult<HashMap<Uuid, PathBuf>> {
90        scan_segment_dirs(&self.segments_path)
91    }
92}
qdrant_edge/edge/read_only/enumerate.rs

qdrant_edge/edge/read_only/
enumerate.rs
