valise/file/query_types.rs
1// This Source Code Form is subject to the terms of the Mozilla Public
2// License, v. 2.0. If a copy of the MPL was not distributed with this
3// file, You can obtain one at https://mozilla.org/MPL/2.0/.
4
5//! Query-input and query-result value types for the `ValiseFile` engine
6//! surface (text / vector / hybrid / time). Re-exported from `file.rs` so
7//! the public path stays `crate::file::<Type>`.
8
9use std::collections::HashSet;
10
11use super::api_types::VectorFidelity;
12use crate::format::catalog::TfMode;
13use crate::format::{
14 CollectionId, EmbeddingSpaceId, FrameId, FusionProfileId, RetrievalProfileId, TextSpaceId,
15 VectorId,
16};
17
18/// Text query input passed to `ValiseFile::query_text`.
19#[derive(Clone, Debug)]
20pub struct TextQuery {
21 pub text_space_id: TextSpaceId,
22 pub query: String,
23 pub algorithm: QueryAlgorithm,
24 /// Optional cap on returned hits. When `Some(k)`, the retriever
25 /// uses a bounded-K heap inside the algorithm — much cheaper than
26 /// sorting the entire touched set and truncating at the boundary.
27 /// `None` keeps the legacy behavior of returning every hit with a
28 /// non-zero score, sorted.
29 pub top_k: Option<usize>,
30 /// Per-term posting-list read budget for the impact-sorted
31 /// vote-then-rerank pipeline used by the cosine scorers. Posting
32 /// lists are sorted by descending term-frequency at decode time;
33 /// the vote phase reads at most `channel_k` entries per query
34 /// term, which bounds the candidate set to `|q| × channel_k`
35 /// regardless of corpus size. The rerank phase then recomputes
36 /// the exact score (full idf / normalization / L2) only for the
37 /// top rerank candidates via binary search on the original
38 /// frame_id-sorted posting list. `None` disables the cap and
39 /// retains exact full-posting scoring.
40 pub channel_k: Option<usize>,
41}
42
43/// Algorithm dispatch for `query_text`. Profile-driven variants pull their
44/// scoring parameters (k1/b, tf_mode/idf_mode/norm_mode, token_source) from
45/// the registered `RetrievalProfileDesc`. Profile-free variants are derived
46/// directly from the canonical primitives per spec §12.0.5 — no profile
47/// registration required.
48#[derive(Clone, Copy, Debug)]
49pub enum QueryAlgorithm {
50 Profile(RetrievalProfileId),
51 /// Profile-free BM25. `k1`/`b` are supplied per query and scoring is
52 /// derived from the canonical df / doc-length / avgdl in
53 /// `TextSpaceState` — no retrieval profile, hence no catalog write on
54 /// the read path (unlike `Profile(Bm25)`). Uses the standard
55 /// Robertson/Sparck-Jones idf. Defaults: `k1 = 1.2`, `b = 0.75`.
56 Bm25 {
57 k1: f32,
58 b: f32,
59 },
60 CountCosine,
61 TfidfCosine {
62 tf_mode: TfMode,
63 },
64 /// Count cosine using `sqrt(doc_length)` as the per-doc norm
65 /// instead of the exact `sqrt(sum_t tf(t,d)^2)`. Lucene-style
66 /// length proxy — trades a small NDCG drift for eliminating the
67 /// index-wide L2 sweep.
68 CountCosineApprox,
69 /// tf-idf cosine with `sqrt(doc_length)` per-doc norm. Same
70 /// trade-off as [`Self::CountCosineApprox`].
71 TfidfCosineApprox {
72 tf_mode: TfMode,
73 },
74 Dice,
75 Overlap,
76 Containment,
77}
78
79/// Default candidate budget for the sign-sketch scan when `channel_k` is
80/// `None`. Fixed and corpus-size-INDEPENDENT on purpose: the sketch's
81/// coverage of the true top-k saturates by a couple thousand candidates at
82/// d≈768 (sweeps 06/14; see `docs/VECTOR_SEARCH.md`), so scaling the budget
83/// with N only multiplies rerank cost for no recall gain. Callers that need a
84/// different operating point (very large N, or higher recall) pass an explicit
85/// `channel_k`.
86pub(crate) const DEFAULT_SKETCH_CANDIDATE_BUDGET: usize = 2048;
87
88/// Input to [`ValiseFile::vector_search`](crate::ValiseFile::vector_search).
89///
90/// Dispatch is implicit per embedding space: a QAM(5,6) space uses the
91/// in-memory sign-sketch scan + QAM-sliding rerank; any other space falls back
92/// to a full brute-force scan through the primary codec. See
93/// `docs/VECTOR_SEARCH.md` for the pipeline and its recall/latency envelope.
94#[derive(Clone, Debug)]
95pub struct VectorSearchQuery {
96 pub embedding_space_id: EmbeddingSpaceId,
97 pub query: Vec<f32>,
98 pub k: usize,
99 /// Candidate budget for the sign-sketch scan: it keeps the `channel_k`
100 /// closest-by-Hamming vectors as rerank candidates. `None` uses
101 /// `max(4 * k, DEFAULT_SKETCH_CANDIDATE_BUDGET)`, clamped to the active
102 /// count. Brute force (non-QAM(5,6) spaces) ignores it.
103 pub channel_k: Option<usize>,
104 /// Optional collection allowlist.
105 pub collection_filter: Option<HashSet<CollectionId>>,
106 /// Rerank fidelity over the sketch candidates. `Lossy` keeps the
107 /// QAM-sliding (i8) scores; `Full` adds an f32 rerank pass over the top
108 /// survivors. Recall is bounded by the codec's reconstruction either way
109 /// (see `docs/VECTOR_SEARCH.md`); `Full` recovers the last ~3 points the
110 /// i8 scorer leaves on the table. Neither changes the candidate set — that
111 /// is `channel_k`'s job.
112 pub fidelity: VectorFidelity,
113}
114
115impl VectorSearchQuery {
116 /// Accurate search: sketch-select the default candidate budget, then run
117 /// the f32 rerank pass (`VectorFidelity::Full`). Leaves `channel_k` unset
118 /// so it uses `DEFAULT_SKETCH_CANDIDATE_BUDGET`, which is sized to reach
119 /// the codec's recall ceiling. Pass an explicit `channel_k` to trade
120 /// recall for latency. This is the engine-level target a higher layer's
121 /// "accurate rerank" lowers to.
122 #[must_use]
123 pub fn accurate(embedding_space_id: EmbeddingSpaceId, query: Vec<f32>, k: usize) -> Self {
124 Self {
125 embedding_space_id,
126 query,
127 k,
128 channel_k: None,
129 collection_filter: None,
130 fidelity: VectorFidelity::Full,
131 }
132 }
133
134 /// Fast search: sketch-select the default candidate budget, keep the
135 /// QAM-sliding (i8) scores, no f32 rerank (`VectorFidelity::Lossy`).
136 /// ~3 points lower recall than [`Self::accurate`] for less rerank work.
137 #[must_use]
138 pub fn fast(embedding_space_id: EmbeddingSpaceId, query: Vec<f32>, k: usize) -> Self {
139 Self {
140 embedding_space_id,
141 query,
142 k,
143 channel_k: None,
144 collection_filter: None,
145 fidelity: VectorFidelity::Lossy,
146 }
147 }
148}
149
150/// Vector hit produced by [`ValiseFile::vector_search`](crate::ValiseFile::vector_search). Score follows
151/// the "smaller = more similar" codec convention.
152#[derive(Clone, Copy, Debug, PartialEq)]
153pub struct VectorHit {
154 pub vector_id: VectorId,
155 pub frame_id: FrameId,
156 pub collection_id: CollectionId,
157 pub score: f32,
158}
159
160/// Per-stage wall-clock breakdown of a [`ValiseFile::vector_search_traced`](crate::ValiseFile::vector_search_traced)
161/// call. Sum of fields ≈ total call cost. `rerank_full` is zero when the
162/// caller did not request [`VectorFidelity::Full`].
163#[derive(Clone, Copy, Debug, Default)]
164pub struct VoteSearchTrace {
165 /// Codec + space resolution.
166 pub preflight: std::time::Duration,
167 /// Unused (sketch query encoding is folded into the scan stage).
168 pub encode: std::time::Duration,
169 /// Unused (the sketch index is derived at file-open, not per query).
170 pub resolve: std::time::Duration,
171 /// Candidate generation: the sign-sketch Hamming scan + counting-sort.
172 pub vote: SketchScanTimings,
173 /// QAM-sliding rerank pass over the candidates.
174 pub rerank_int: std::time::Duration,
175 /// Optional f32 re-scoring pass. Zero unless `query.fidelity ==
176 /// VectorFidelity::Full`.
177 pub rerank_full: std::time::Duration,
178}
179
180/// Sub-stage timings for the sign-sketch candidate scan.
181#[derive(Clone, Copy, Debug, Default)]
182pub struct SketchScanTimings {
183 /// Fused Hamming scan + histogram + counting-sort threshold.
184 pub accumulate: std::time::Duration,
185}
186
187/// Input to [`ValiseFile::time_range_query`](crate::ValiseFile::time_range_query), spec §17. `from` and `to` are
188/// inclusive Unix epoch seconds. Empty range (`from > to`) yields no hits.
189#[derive(Clone, Copy, Debug)]
190pub struct TimeQuery {
191 pub from: i64,
192 pub to: i64,
193 /// Restrict to one collection. `None` returns hits from all collections.
194 pub collection_id: Option<CollectionId>,
195}
196
197/// Input to [`ValiseFile::query_hybrid`](crate::ValiseFile::query_hybrid), spec §18.3.
198///
199/// One text channel (running an indexed retrieval profile) and one vector
200/// channel are fused under `fusion_profile_id`. Either channel may be left
201/// unset, in which case the fused score is just the other channel.
202#[derive(Clone, Debug)]
203pub struct HybridQuery {
204 pub fusion_profile_id: FusionProfileId,
205 /// Text channel: omit to skip text retrieval entirely.
206 pub text: Option<HybridTextChannel>,
207 /// Vector channel: omit to skip vector retrieval entirely.
208 pub vector: Option<HybridVectorChannel>,
209 pub k: usize,
210 /// Per-channel candidate budget. Each channel pulls this many hits
211 /// before fusion picks the top-k.
212 pub channel_k: usize,
213}
214
215#[derive(Clone, Debug)]
216pub struct HybridTextChannel {
217 pub text_space_id: TextSpaceId,
218 pub query: String,
219 pub algorithm: QueryAlgorithm,
220}
221
222#[derive(Clone, Debug)]
223pub struct HybridVectorChannel {
224 pub embedding_space_id: EmbeddingSpaceId,
225 pub query: Vec<f32>,
226 pub ef: Option<usize>,
227 pub fidelity: VectorFidelity,
228}
229
230/// Fused hit produced by [`ValiseFile::query_hybrid`](crate::ValiseFile::query_hybrid). `score` follows the
231/// "higher = better" convention (vector distances are flipped before
232/// fusion so all channels share direction).
233#[derive(Clone, Copy, Debug, PartialEq)]
234pub struct HybridHit {
235 pub frame_id: FrameId,
236 pub score: f32,
237}