1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
//! Text-index specifics for the embedded API: declaring a multi-field
//! text index, and gathering the corpus statistics a global BM25 scores
//! against. Split from `ops_index.rs` for the 500-LOC house rule (a
//! `#[path]` child module, so it reaches `Store`'s private fields).
use kevy_index::{IndexKind, IndexSpec, ValType};
use super::sync_segs;
use crate::TokenPositions;
use crate::store::{Store, lock_write};
use crate::{KevyError, KevyResult};
impl Store {
/// A multi-field text index over several weighted hash fields — what
/// the wire creates with `ON FIELDS f1 f2 WEIGHTS 2 1`, and what `IN`
/// scopes to. A weight scales that field's term frequencies; 1.0 is
/// neutral.
///
/// `positions` decides whether token offsets are recorded, so phrase
/// queries can verify adjacency, at the cost of the positional
/// side-channel's memory.
/// `values` names hash fields stored per document with the type their
/// bytes compare as — what `FILTER` reads. An index that never
/// filters declares none.
pub fn idx_create_text(
&self,
name: &[u8],
prefix: &[u8],
fields: &[(&[u8], f32)],
positions: TokenPositions,
values: &[(&[u8], ValType)],
) -> KevyResult<()> {
if prefix.is_empty() {
return Err(KevyError::InvalidInput("empty prefix".into()));
}
if fields.is_empty() {
return Err(KevyError::InvalidInput("a text index needs at least one field".into()));
}
let fields =
fields.iter().map(|(f, w)| kevy_index::FieldSpec::new(*f).with_weight(*w)).collect();
let values =
values.iter().map(|(n, ty)| kevy_index::ValueSpec::new(*n).with_type(*ty)).collect();
let spec = IndexSpec::builder(name, prefix, IndexKind::Text, ValType::Str)
.with_fields(fields)
.with_positions(matches!(positions, TokenPositions::Record))
.with_values(values);
let spec = crate::ops_index::built(spec)?;
self.catalog_change(|| self.register_spec(spec))
}
/// Corpus-wide BM25 statistics for one query, over its field scope.
///
/// Every shard reports its own document count, its token total and,
/// per query term, its local document frequency; summing them is what
/// makes the scores comparable across shards. Restricted to a field
/// scope, each of those numbers describes those fields rather than
/// whole documents, so a scoped query's `avgdl` is the average length
/// *of the named fields*.
pub(crate) fn text_corpus_stats_in(
&self,
name: &[u8],
text: &[u8],
typo: u32,
scope: &[usize],
) -> KevyResult<kevy_text::CorpusStats> {
let (mut n_docs, mut total_len, mut found) = (0f64, 0u64, false);
// Accumulated per shard from `query_df_in`, which expands `word*`
// prefixes against that shard's dictionary — so the df map ends up
// keyed by the union of every shard's query terms and prefix
// expansions, each summed to a global df.
let mut df: std::collections::HashMap<Vec<u8>, u32> = std::collections::HashMap::new();
for shard in self.shards.iter() {
let mut g = lock_write(shard);
let inner = &mut *g;
sync_segs(&self.indexes, &mut inner.idx_segs, &mut inner.store);
if let Some((_, ts)) = inner.idx_segs.text.iter().find(|(s, _)| s.name() == name) {
found = true;
n_docs += ts.docs() as f64;
total_len += ts.total_len_in(scope);
let opts = kevy_text::QueryOpts::default().with_typo(typo).with_fields(scope);
let tokdf = ts.query_df_in(text, opts);
// The shard's frozen buckets are pass-1 contributors
// like any other shard — the server seam's mirror.
#[cfg(not(target_arch = "wasm32"))]
super::text_cold::fold_cold_stats(
inner.idx_segs.cold_text_of(name),
&tokdf,
&mut n_docs,
&mut total_len,
&mut df,
);
for (t, d) in tokdf {
*df.entry(t).or_insert(0) += d;
}
}
}
if !found {
return Err(KevyError::NotFound("no such text index".into()));
}
let avgdl = if n_docs > 0.0 { total_len as f64 / n_docs } else { 0.0 };
Ok(kevy_text::CorpusStats::new(n_docs, avgdl, df))
}
}